{
  "type": "excalidraw",
  "version": 2,
  "source": "http://127.0.0.1:8765",
  "elements": [
    {
      "id": "W73OucrcZ9WuvzUdxcmBT",
      "type": "text",
      "x": 0,
      "y": 0,
      "width": 480.86956787109375,
      "height": 37.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a0",
      "roundness": null,
      "seed": 1,
      "version": 2,
      "versionNonce": 262171425,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "How GPT is trained: four stages",
      "fontSize": 30,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "How GPT is trained: four stages",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "Ma_fJWsse3_6fBxFju6yA",
      "type": "rectangle",
      "x": 0,
      "y": 60,
      "width": 270,
      "height": 380,
      "angle": 0,
      "strokeColor": "#1971c2",
      "backgroundColor": "#e7f5ff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a1",
      "roundness": null,
      "seed": 2,
      "version": 2,
      "versionNonce": 155718511,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false
    },
    {
      "id": "m0PpjD1X1lQEnRTbR6als",
      "type": "text",
      "x": 48.54993438720703,
      "y": 76,
      "width": 172.90013122558594,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#1971c2",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a2",
      "roundness": null,
      "seed": 3,
      "version": 2,
      "versionNonce": 495476481,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "1 · Pretraining",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "1 · Pretraining",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "cwkRs4Y8xG1kyG--2y4hW",
      "type": "text",
      "x": 16,
      "y": 130,
      "width": 215.4226531982422,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a3",
      "roundness": null,
      "seed": 4,
      "version": 2,
      "versionNonce": 619826575,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "data: 300 B tokens of web\ntext, books, Wikipedia",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "data: 300 B tokens of web\ntext, books, Wikipedia",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "Iguqh1wsXWnxZ87ukgq_M",
      "type": "line",
      "x": 16,
      "y": 200,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a4",
      "roundness": null,
      "seed": 5,
      "version": 2,
      "versionNonce": 906417889,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "UEctPxivRzEQ3rV_Rysi-",
      "type": "text",
      "x": 16,
      "y": 214,
      "width": 131.5251922607422,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#1971c2",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a5",
      "roundness": null,
      "seed": 6,
      "version": 2,
      "versionNonce": 866075567,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "loss: next-token\ncross-entropy",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "loss: next-token\ncross-entropy",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "Vy1H_yQUZtk2rTvSdSMQI",
      "type": "line",
      "x": 16,
      "y": 284,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a6",
      "roundness": null,
      "seed": 7,
      "version": 2,
      "versionNonce": 1269519041,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "_DNPNaMYjnOtvEQBhC6l9",
      "type": "text",
      "x": 16,
      "y": 298,
      "width": 52.19212341308594,
      "height": 21.6,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a7",
      "roundness": null,
      "seed": 8,
      "version": 2,
      "versionNonce": 360248783,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "output:",
      "fontSize": 16,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "output:",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "JQSnLPC3nXBOBLVSyhZQC",
      "type": "text",
      "x": 16,
      "y": 324,
      "width": 158.7173309326172,
      "height": 52.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a8",
      "roundness": null,
      "seed": 9,
      "version": 2,
      "versionNonce": 1393906337,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "base model\n(GPT-3, 175 B)",
      "fontSize": 21,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "base model\n(GPT-3, 175 B)",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "55OTBGrL6RiSB6C39rVHB",
      "type": "arrow",
      "x": 274,
      "y": 250,
      "width": 22,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a9",
      "roundness": null,
      "seed": 10,
      "version": 2,
      "versionNonce": 295134191,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "points": [
        [
          0.5,
          0
        ],
        [
          21.5,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "uYbiTY1i7kCWccUb2_B91",
      "type": "rectangle",
      "x": 300,
      "y": 60,
      "width": 270,
      "height": 380,
      "angle": 0,
      "strokeColor": "#2f9e44",
      "backgroundColor": "#ebfbee",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aA",
      "roundness": null,
      "seed": 11,
      "version": 2,
      "versionNonce": 16012929,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false
    },
    {
      "id": "QKQn6Y59z8XKTfcJtACDU",
      "type": "text",
      "x": 385.0799560546875,
      "y": 76,
      "width": 99.840087890625,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#2f9e44",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aB",
      "roundness": null,
      "seed": 12,
      "version": 2,
      "versionNonce": 2013466127,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "2 · SFT",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "2 · SFT",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "nt93IF_1BnExwqqqi-2xj",
      "type": "text",
      "x": 316,
      "y": 130,
      "width": 207.0167236328125,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aC",
      "roundness": null,
      "seed": 13,
      "version": 2,
      "versionNonce": 1208475233,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "text": "data: ~13 k prompts with\nlabeler-written answers",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "data: ~13 k prompts with\nlabeler-written answers",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "l6dTaxzpo5J-v7yRVVVgh",
      "type": "line",
      "x": 316,
      "y": 200,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aD",
      "roundness": null,
      "seed": 14,
      "version": 2,
      "versionNonce": 62157871,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168135,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "lN7_zM-Wu2j5oB2VeGhOw",
      "type": "text",
      "x": 316,
      "y": 214,
      "width": 159.19100952148438,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#2f9e44",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aE",
      "roundness": null,
      "seed": 15,
      "version": 2,
      "versionNonce": 387934785,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "loss: same, on\nanswer tokens only",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "loss: same, on\nanswer tokens only",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "-jMNw4wYtShE-3RWbjfnU",
      "type": "line",
      "x": 316,
      "y": 284,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aF",
      "roundness": null,
      "seed": 16,
      "version": 2,
      "versionNonce": 2090079823,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "rBWbFdj4el2XvmdAPzx3q",
      "type": "text",
      "x": 316,
      "y": 298,
      "width": 52.19212341308594,
      "height": 21.6,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aG",
      "roundness": null,
      "seed": 17,
      "version": 2,
      "versionNonce": 1461352993,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "output:",
      "fontSize": 16,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "output:",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "ITMMouPMmY4xKscGjYfqW",
      "type": "text",
      "x": 316,
      "y": 324,
      "width": 108.56951904296875,
      "height": 26.25,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aH",
      "roundness": null,
      "seed": 18,
      "version": 2,
      "versionNonce": 427455599,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "SFT model",
      "fontSize": 21,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "SFT model",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "UlkljJTxxdFLhJptLdamV",
      "type": "arrow",
      "x": 574,
      "y": 250,
      "width": 22,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aI",
      "roundness": null,
      "seed": 19,
      "version": 2,
      "versionNonce": 686329345,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0.5,
          0
        ],
        [
          21.5,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "PW_qUwz4m85VCpamky6q6",
      "type": "rectangle",
      "x": 600,
      "y": 60,
      "width": 270,
      "height": 380,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#fff4e6",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aJ",
      "roundness": null,
      "seed": 20,
      "version": 2,
      "versionNonce": 573574799,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false
    },
    {
      "id": "rL_i9FNguTx_PhLZZZhvf",
      "type": "text",
      "x": 628.8289031982422,
      "y": 76,
      "width": 212.34219360351562,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aK",
      "roundness": null,
      "seed": 21,
      "version": 2,
      "versionNonce": 1669932513,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "3 · Reward model",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "3 · Reward model",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "2EPfVust5k3KdbWEO8PG4",
      "type": "text",
      "x": 616,
      "y": 130,
      "width": 169.73895263671875,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aL",
      "roundness": null,
      "seed": 22,
      "version": 2,
      "versionNonce": 1366123695,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "data: ~33 k prompts,\n4–9 answers ranked",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "data: ~33 k prompts,\n4–9 answers ranked",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "NN37BruQ2fgt8DNgTtOlR",
      "type": "line",
      "x": 616,
      "y": 200,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aM",
      "roundness": null,
      "seed": 23,
      "version": 2,
      "versionNonce": 1376502209,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "j05WenH9vI-KqacORmPeu",
      "type": "text",
      "x": 616,
      "y": 214,
      "width": 177.44406127929688,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aN",
      "roundness": null,
      "seed": 24,
      "version": 2,
      "versionNonce": 1994061519,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "loss: pairwise\n−log σ(r_win − r_lose)",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "loss: pairwise\n−log σ(r_win − r_lose)",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "GH6WhHVq9mo1asyyTtIGZ",
      "type": "line",
      "x": 616,
      "y": 284,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aO",
      "roundness": null,
      "seed": 25,
      "version": 2,
      "versionNonce": 831512993,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "uASIFqbsKdnpOdBzQdcRm",
      "type": "text",
      "x": 616,
      "y": 298,
      "width": 52.19212341308594,
      "height": 21.6,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aP",
      "roundness": null,
      "seed": 26,
      "version": 2,
      "versionNonce": 1494303983,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "output:",
      "fontSize": 16,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "output:",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "MPgy90_o-GrE9bwOOjNUl",
      "type": "text",
      "x": 616,
      "y": 324,
      "width": 191.05714416503906,
      "height": 52.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aQ",
      "roundness": null,
      "seed": 27,
      "version": 2,
      "versionNonce": 1866794369,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "reward model\n(6 B, frozen later)",
      "fontSize": 21,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "reward model\n(6 B, frozen later)",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "wldLceUBYINPC9RR1a9Kg",
      "type": "arrow",
      "x": 874,
      "y": 250,
      "width": 22,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aR",
      "roundness": null,
      "seed": 28,
      "version": 2,
      "versionNonce": 1469632271,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0.5,
          0
        ],
        [
          21.5,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "pVIp7PQ-mKVPqcSXKqc-7",
      "type": "rectangle",
      "x": 900,
      "y": 60,
      "width": 270,
      "height": 380,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "#f3f0ff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aS",
      "roundness": null,
      "seed": 29,
      "version": 2,
      "versionNonce": 644525409,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false
    },
    {
      "id": "_Hr0Ar1lF27fFYi8VSt3p",
      "type": "text",
      "x": 938.0979232788086,
      "y": 76,
      "width": 193.8041534423828,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aT",
      "roundness": null,
      "seed": 30,
      "version": 2,
      "versionNonce": 1290409263,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "4 · RLHF (PPO)",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "4 · RLHF (PPO)",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "FOjLdiiKYM-dMgVDEMsKw",
      "type": "text",
      "x": 916,
      "y": 130,
      "width": 169.73895263671875,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aU",
      "roundness": null,
      "seed": 31,
      "version": 2,
      "versionNonce": 1582324033,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "data: ~31 k prompts,\nno answers needed",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "data: ~31 k prompts,\nno answers needed",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "sFvp7UpoYiuc3MO_TBkIp",
      "type": "line",
      "x": 916,
      "y": 200,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aV",
      "roundness": null,
      "seed": 32,
      "version": 2,
      "versionNonce": 812488527,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "fevRZyQnUf39N8z7T33yO",
      "type": "text",
      "x": 916,
      "y": 214,
      "width": 142.73910522460938,
      "height": 48.6,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aW",
      "roundness": null,
      "seed": 33,
      "version": 2,
      "versionNonce": 139823393,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "goal: max reward\n− β · KL to SFT",
      "fontSize": 18,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "goal: max reward\n− β · KL to SFT",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "LjPFeMpgKCXbvY-Q6PRSn",
      "type": "line",
      "x": 916,
      "y": 284,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#dee2e6",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 1,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aX",
      "roundness": null,
      "seed": 34,
      "version": 2,
      "versionNonce": 2021381487,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          238,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "ZISQi2gcQima1DfRGHNDh",
      "type": "text",
      "x": 916,
      "y": 298,
      "width": 52.19212341308594,
      "height": 21.6,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aY",
      "roundness": null,
      "seed": 35,
      "version": 2,
      "versionNonce": 1038728449,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "output:",
      "fontSize": 16,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "output:",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "F3wElnxuHdlaK_6P0f2gs",
      "type": "text",
      "x": 916,
      "y": 324,
      "width": 219.00804138183594,
      "height": 52.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aZ",
      "roundness": null,
      "seed": 36,
      "version": 2,
      "versionNonce": 1012748175,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "InstructGPT /\nChatGPT-style model",
      "fontSize": 21,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "InstructGPT /\nChatGPT-style model",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "MIR8bcHK7TpILMavXfQG1",
      "type": "text",
      "x": 0,
      "y": 462,
      "width": 977.6336669921875,
      "height": 22.950000000000003,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aa",
      "roundness": null,
      "seed": 37,
      "version": 2,
      "versionNonce": 1049392353,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791092168136,
      "link": null,
      "locked": false,
      "text": "Stages 2–4 are the InstructGPT recipe (Ouyang et al. 2022); stage 1 is GPT-3 (Brown et al. 2020). Sizes are from those papers.",
      "fontSize": 17,
      "fontFamily": 6,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Stages 2–4 are the InstructGPT recipe (Ouyang et al. 2022); stage 1 is GPT-3 (Brown et al. 2020). Sizes are from those papers.",
      "autoResize": true,
      "lineHeight": 1.35
    }
  ],
  "appState": {
    "viewBackgroundColor": "#ffffff"
  },
  "files": {}
}