feat: rewrite d2c to not require token (#8269)

excalidraw-app/App.tsx

@@ -22,7 +22,6 @@ import { t } from "../packages/excalidraw/i18n";
 import {
   Excalidraw,
   LiveCollaborationTrigger,
-  TTDDialog,
   TTDDialogTrigger,
   StoreAction,
   reconcileElements,
@@ -121,6 +120,7 @@ import {
 import { appThemeAtom, useHandleAppTheme } from "./useHandleAppTheme";
 import { getPreferredLanguage } from "./app-language/language-detector";
 import { useAppLangCode } from "./app-language/language-state";
+import { AIComponents } from "./components/AI";
 
 polyfill();
 
@@ -846,63 +846,8 @@ const ExcalidrawWrapper = () => {
           )}
         </OverwriteConfirmDialog>
         <AppFooter />
-        <TTDDialog
+        {excalidrawAPI && <AIComponents excalidrawAPI={excalidrawAPI} />}
-          onTextSubmit={async (input) => {
-            try {
-              const response = await fetch(
-                `${
-                  import.meta.env.VITE_APP_AI_BACKEND
-                }/v1/ai/text-to-diagram/generate`,
-                {
-                  method: "POST",
-                  headers: {
-                    Accept: "application/json",
-                    "Content-Type": "application/json",
-                  },
-                  body: JSON.stringify({ prompt: input }),
-                },
-              );
 
-              const rateLimit = response.headers.has("X-Ratelimit-Limit")
-                ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
-                : undefined;
-
-              const rateLimitRemaining = response.headers.has(
-                "X-Ratelimit-Remaining",
-              )
-                ? parseInt(
-                    response.headers.get("X-Ratelimit-Remaining") || "0",
-                    10,
-                  )
-                : undefined;
-
-              const json = await response.json();
-
-              if (!response.ok) {
-                if (response.status === 429) {
-                  return {
-                    rateLimit,
-                    rateLimitRemaining,
-                    error: new Error(
-                      "Too many requests today, please try again tomorrow!",
-                    ),
-                  };
-                }
-
-                throw new Error(json.message || "Generation failed...");
-              }
-
-              const generatedResponse = json.generatedResponse;
-              if (!generatedResponse) {
-                throw new Error("Generation failed...");
-              }
-
-              return { generatedResponse, rateLimit, rateLimitRemaining };
-            } catch (err: any) {
-              throw new Error("Request failed");
-            }
-          }}
-        />
         <TTDDialogTrigger />
         {isCollaborating && isOffline && (
           <div className="collab-offline-warning">

excalidraw-app/components/AI.tsx

@@ -0,0 +1,159 @@
+import type { ExcalidrawImperativeAPI } from "../../packages/excalidraw/types";
+import {
+  DiagramToCodePlugin,
+  exportToBlob,
+  getTextFromElements,
+  MIME_TYPES,
+  TTDDialog,
+} from "../../packages/excalidraw";
+import { getDataURL } from "../../packages/excalidraw/data/blob";
+import { safelyParseJSON } from "../../packages/excalidraw/utils";
+
+export const AIComponents = ({
+  excalidrawAPI,
+}: {
+  excalidrawAPI: ExcalidrawImperativeAPI;
+}) => {
+  return (
+    <>
+      <DiagramToCodePlugin
+        generate={async ({ frame, children }) => {
+          const appState = excalidrawAPI.getAppState();
+
+          const blob = await exportToBlob({
+            elements: children,
+            appState: {
+              ...appState,
+              exportBackground: true,
+              viewBackgroundColor: appState.viewBackgroundColor,
+            },
+            exportingFrame: frame,
+            files: excalidrawAPI.getFiles(),
+            mimeType: MIME_TYPES.jpg,
+          });
+
+          const dataURL = await getDataURL(blob);
+
+          const textFromFrameChildren = getTextFromElements(children);
+
+          const response = await fetch(
+            `${
+              import.meta.env.VITE_APP_AI_BACKEND
+            }/v1/ai/diagram-to-code/generate`,
+            {
+              method: "POST",
+              headers: {
+                Accept: "application/json",
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({
+                texts: textFromFrameChildren,
+                image: dataURL,
+                theme: appState.theme,
+              }),
+            },
+          );
+
+          if (!response.ok) {
+            const text = await response.text();
+            const errorJSON = safelyParseJSON(text);
+
+            if (!errorJSON) {
+              throw new Error(text);
+            }
+
+            if (errorJSON.statusCode === 429) {
+              return {
+                html: `<html>
+                <body style="margin: 0; text-align: center">
+                <div style="display: flex; align-items: center; justify-content: center; flex-direction: column; height: 100vh; padding: 0 60px">
+                  <div style="color:red">Too many requests today,</br>please try again tomorrow!</div>
+                  </br>
+                  </br>
+                  <div>You can also try <a href="${
+                    import.meta.env.VITE_APP_PLUS_LP
+                  }/plus?utm_source=excalidraw&utm_medium=app&utm_content=d2c" target="_blank" rel="noreferrer noopener">Excalidraw+</a> to get more requests.</div>
+                </div>
+                </body>
+                </html>`,
+              };
+            }
+
+            throw new Error(errorJSON.message || text);
+          }
+
+          try {
+            const { html } = await response.json();
+
+            if (!html) {
+              throw new Error("Generation failed (invalid response)");
+            }
+            return {
+              html,
+            };
+          } catch (error: any) {
+            throw new Error("Generation failed (invalid response)");
+          }
+        }}
+      />
+
+      <TTDDialog
+        onTextSubmit={async (input) => {
+          try {
+            const response = await fetch(
+              `${
+                import.meta.env.VITE_APP_AI_BACKEND
+              }/v1/ai/text-to-diagram/generate`,
+              {
+                method: "POST",
+                headers: {
+                  Accept: "application/json",
+                  "Content-Type": "application/json",
+                },
+                body: JSON.stringify({ prompt: input }),
+              },
+            );
+
+            const rateLimit = response.headers.has("X-Ratelimit-Limit")
+              ? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
+              : undefined;
+
+            const rateLimitRemaining = response.headers.has(
+              "X-Ratelimit-Remaining",
+            )
+              ? parseInt(
+                  response.headers.get("X-Ratelimit-Remaining") || "0",
+                  10,
+                )
+              : undefined;
+
+            const json = await response.json();
+
+            if (!response.ok) {
+              if (response.status === 429) {
+                return {
+                  rateLimit,
+                  rateLimitRemaining,
+                  error: new Error(
+                    "Too many requests today, please try again tomorrow!",
+                  ),
+                };
+              }
+
+              throw new Error(json.message || "Generation failed...");
+            }
+
+            const generatedResponse = json.generatedResponse;
+            if (!generatedResponse) {
+              throw new Error("Generation failed...");
+            }
+
+            return { generatedResponse, rateLimit, rateLimitRemaining };
+          } catch (err: any) {
+            throw new Error("Request failed");
+          }
+        }}
+      />
+    </>
+  );
+};

packages/excalidraw/actions/actionClipboard.tsx

@@ -10,7 +10,7 @@ import {
 } from "../clipboard";
 import { actionDeleteSelected } from "./actionDeleteSelected";
 import { exportCanvas, prepareElementsForExport } from "../data/index";
-import { isTextElement } from "../element";
+import { getTextFromElements, isTextElement } from "../element";
 import { t } from "../i18n";
 import { isFirefox } from "../constants";
 import { DuplicateIcon, cutIcon, pngIcon, svgIcon } from "../components/icons";
@@ -239,16 +239,8 @@ export const copyText = register({
       includeBoundTextElement: true,
     });
 
-    const text = selectedElements
-      .reduce((acc: string[], element) => {
-        if (isTextElement(element)) {
-          acc.push(element.text);
-        }
-        return acc;
-      }, [])
-      .join("\n\n");
     try {
-      copyTextToSystemClipboard(text);
+      copyTextToSystemClipboard(getTextFromElements(selectedElements));
     } catch (e) {
       throw new Error(t("errors.copyToSystemClipboardFailed"));
     }

packages/excalidraw/components/Actions.tsx

@@ -45,7 +45,6 @@ import {
   frameToolIcon,
   mermaidLogoIcon,
   laserPointerToolIcon,
-  OpenAIIcon,
   MagicIcon,
 } from "./icons";
 import { KEYS } from "../keys";
@@ -400,7 +399,7 @@ export const ShapesSwitcher = ({
           >
             {t("toolBar.mermaidToExcalidraw")}
           </DropdownMenu.Item>
-          {app.props.aiEnabled !== false && (
+          {app.props.aiEnabled !== false && app.plugins.diagramToCode && (
             <>
               <DropdownMenu.Item
                 onSelect={() => app.onMagicframeToolSelect()}
@@ -410,20 +409,6 @@ export const ShapesSwitcher = ({
                 {t("toolBar.magicframe")}
                 <DropdownMenu.Item.Badge>AI</DropdownMenu.Item.Badge>
               </DropdownMenu.Item>
-              <DropdownMenu.Item
-                onSelect={() => {
-                  trackEvent("ai", "open-settings", "d2c");
-                  app.setOpenDialog({
-                    name: "settings",
-                    source: "settings",
-                    tab: "diagram-to-code",
-                  });
-                }}
-                icon={OpenAIIcon}
-                data-testid="toolbar-magicSettings"
-              >
-                {t("toolBar.magicSettings")}
-              </DropdownMenu.Item>
             </>
           )}
         </DropdownMenu.Content>

packages/excalidraw/components/App.tsx

@@ -83,7 +83,6 @@ import {
   ZOOM_STEP,
   POINTER_EVENTS,
   TOOL_TYPE,
-  EDITOR_LS_KEYS,
   isIOS,
   supportsResizeObserver,
   DEFAULT_COLLISION_THRESHOLD,
@@ -183,6 +182,7 @@ import type {
   ExcalidrawIframeElement,
   ExcalidrawEmbeddableElement,
   Ordered,
+  MagicGenerationData,
   ExcalidrawNonSelectionElement,
   ExcalidrawArrowElement,
 } from "../element/types";
@@ -257,6 +257,7 @@ import type {
   UnsubscribeCallback,
   EmbedsValidationStatus,
   ElementsPendingErasure,
+  GenerateDiagramToCode,
   NullableGridSize,
 } from "../types";
 import {
@@ -405,13 +406,9 @@ import {
 } from "../cursor";
 import { Emitter } from "../emitter";
 import { ElementCanvasButtons } from "../element/ElementCanvasButtons";
-import type { MagicCacheData } from "../data/magic";
-import { diagramToHTML } from "../data/magic";
-import { exportToBlob } from "../../utils/export";
 import { COLOR_PALETTE } from "../colors";
 import { ElementCanvasButton } from "./MagicButton";
 import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
-import { EditorLocalStorage } from "../data/EditorLocalStorage";
 import FollowMode from "./FollowMode/FollowMode";
 import { Store, StoreAction } from "../store";
 import { AnimationFrameHandler } from "../animation-frame-handler";
@@ -1018,7 +1015,7 @@ class App extends React.Component<AppProps, AppState> {
           if (isIframeElement(el)) {
             src = null;
 
-            const data: MagicCacheData = (el.customData?.generationData ??
+            const data: MagicGenerationData = (el.customData?.generationData ??
               this.magicGenerations.get(el.id)) || {
               status: "error",
               message: "No generation data",
@@ -1559,10 +1556,6 @@ class App extends React.Component<AppProps, AppState> {
                           }
                           app={this}
                           isCollaborating={this.props.isCollaborating}
-                          openAIKey={this.OPENAI_KEY}
-                          isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED}
-                          onOpenAIAPIKeyChange={this.onOpenAIKeyChange}
-                          onMagicSettingsConfirm={this.onMagicSettingsConfirm}
                         >
                           {this.props.children}
                         </LayerUI>
@@ -1807,7 +1800,7 @@ class App extends React.Component<AppProps, AppState> {
 
   private magicGenerations = new Map<
     ExcalidrawIframeElement["id"],
-    MagicCacheData
+    MagicGenerationData
   >();
 
   private updateMagicGeneration = ({
@@ -1815,7 +1808,7 @@ class App extends React.Component<AppProps, AppState> {
     data,
   }: {
     frameElement: ExcalidrawIframeElement;
-    data: MagicCacheData;
+    data: MagicGenerationData;
   }) => {
     if (data.status === "pending") {
       // We don't wanna persist pending state to storage. It should be in-app
@@ -1838,31 +1831,26 @@ class App extends React.Component<AppProps, AppState> {
     this.triggerRender();
   };
 
-  private getTextFromElements(elements: readonly ExcalidrawElement[]) {
+  public plugins: {
-    const text = elements
+    diagramToCode?: {
-      .reduce((acc: string[], element) => {
+      generate: GenerateDiagramToCode;
-        if (isTextElement(element)) {
+    };
-          acc.push(element.text);
+  } = {};
-        }
+
-        return acc;
+  public setPlugins(plugins: Partial<App["plugins"]>) {
-      }, [])
+    Object.assign(this.plugins, plugins);
-      .join("\n\n");
-    return text;
   }
 
   private async onMagicFrameGenerate(
     magicFrame: ExcalidrawMagicFrameElement,
     source: "button" | "upstream",
   ) {
-    if (!this.OPENAI_KEY) {
+    const generateDiagramToCode = this.plugins.diagramToCode?.generate;
+
+    if (!generateDiagramToCode) {
       this.setState({
-        openDialog: {
+        errorMessage: "No diagram to code plugin found",
-          name: "settings",
-          tab: "diagram-to-code",
-          source: "generation",
-        },
       });
-      trackEvent("ai", "generate (missing key)", "d2c");
       return;
     }
 
@@ -1901,68 +1889,50 @@ class App extends React.Component<AppProps, AppState> {
       selectedElementIds: { [frameElement.id]: true },
     });
 
-    const blob = await exportToBlob({
-      elements: this.scene.getNonDeletedElements(),
-      appState: {
-        ...this.state,
-        exportBackground: true,
-        viewBackgroundColor: this.state.viewBackgroundColor,
-      },
-      exportingFrame: magicFrame,
-      files: this.files,
-    });
-
-    const dataURL = await getDataURL(blob);
-
-    const textFromFrameChildren = this.getTextFromElements(magicFrameChildren);
-
     trackEvent("ai", "generate (start)", "d2c");
+    try {
+      const { html } = await generateDiagramToCode({
+        frame: magicFrame,
+        children: magicFrameChildren,
+      });
 
-    const result = await diagramToHTML({
+      trackEvent("ai", "generate (success)", "d2c");
-      image: dataURL,
-      apiKey: this.OPENAI_KEY,
-      text: textFromFrameChildren,
-      theme: this.state.theme,
-    });
 
-    if (!result.ok) {
+      if (!html.trim()) {
+        this.updateMagicGeneration({
+          frameElement,
+          data: {
+            status: "error",
+            code: "ERR_OAI",
+            message: "Nothing genereated :(",
+          },
+        });
+        return;
+      }
+
+      const parsedHtml =
+        html.includes("<!DOCTYPE html>") && html.includes("</html>")
+          ? html.slice(
+              html.indexOf("<!DOCTYPE html>"),
+              html.indexOf("</html>") + "</html>".length,
+            )
+          : html;
+
+      this.updateMagicGeneration({
+        frameElement,
+        data: { status: "done", html: parsedHtml },
+      });
+    } catch (error: any) {
       trackEvent("ai", "generate (failed)", "d2c");
-      console.error(result.error);
       this.updateMagicGeneration({
         frameElement,
         data: {
           status: "error",
           code: "ERR_OAI",
-          message: result.error?.message || "Unknown error during generation",
+          message: error.message || "Unknown error during generation",
         },
       });
-      return;
     }
-    trackEvent("ai", "generate (success)", "d2c");
-
-    if (result.choices[0].message.content == null) {
-      this.updateMagicGeneration({
-        frameElement,
-        data: {
-          status: "error",
-          code: "ERR_OAI",
-          message: "Nothing genereated :(",
-        },
-      });
-      return;
-    }
-
-    const message = result.choices[0].message.content;
-
-    const html = message.slice(
-      message.indexOf("<!DOCTYPE html>"),
-      message.indexOf("</html>") + "</html>".length,
-    );
-
-    this.updateMagicGeneration({
-      frameElement,
-      data: { status: "done", html },
-    });
   }
 
   private onIframeSrcCopy(element: ExcalidrawIframeElement) {
@@ -1976,70 +1946,7 @@ class App extends React.Component<AppProps, AppState> {
     }
   }
 
-  private OPENAI_KEY: string | null = EditorLocalStorage.get(
-    EDITOR_LS_KEYS.OAI_API_KEY,
-  );
-  private OPENAI_KEY_IS_PERSISTED: boolean =
-    EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false;
-
-  private onOpenAIKeyChange = (
-    openAIKey: string | null,
-    shouldPersist: boolean,
-  ) => {
-    this.OPENAI_KEY = openAIKey || null;
-    if (shouldPersist) {
-      const didPersist = EditorLocalStorage.set(
-        EDITOR_LS_KEYS.OAI_API_KEY,
-        openAIKey,
-      );
-      this.OPENAI_KEY_IS_PERSISTED = didPersist;
-    } else {
-      this.OPENAI_KEY_IS_PERSISTED = false;
-    }
-  };
-
-  private onMagicSettingsConfirm = (
-    apiKey: string,
-    shouldPersist: boolean,
-    source: "tool" | "generation" | "settings",
-  ) => {
-    this.OPENAI_KEY = apiKey || null;
-    this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist);
-
-    if (source === "settings") {
-      return;
-    }
-
-    const selectedElements = this.scene.getSelectedElements({
-      selectedElementIds: this.state.selectedElementIds,
-    });
-
-    if (apiKey) {
-      if (selectedElements.length) {
-        this.onMagicframeToolSelect();
-      } else {
-        this.setActiveTool({ type: "magicframe" });
-      }
-    } else if (!isMagicFrameElement(selectedElements[0])) {
-      // even if user didn't end up setting api key, let's pick the tool
-      // so they can draw up a frame and move forward
-      this.setActiveTool({ type: "magicframe" });
-    }
-  };
-
   public onMagicframeToolSelect = () => {
-    if (!this.OPENAI_KEY) {
-      this.setState({
-        openDialog: {
-          name: "settings",
-          tab: "diagram-to-code",
-          source: "tool",
-        },
-      });
-      trackEvent("ai", "tool-select (missing key)", "d2c");
-      return;
-    }
-
     const selectedElements = this.scene.getSelectedElements({
       selectedElementIds: this.state.selectedElementIds,
     });

packages/excalidraw/components/DiagramToCodePlugin/DiagramToCodePlugin.tsx

@@ -0,0 +1,17 @@
+import { useLayoutEffect } from "react";
+import { useApp } from "../App";
+import type { GenerateDiagramToCode } from "../../types";
+
+export const DiagramToCodePlugin = (props: {
+  generate: GenerateDiagramToCode;
+}) => {
+  const app = useApp();
+
+  useLayoutEffect(() => {
+    app.setPlugins({
+      diagramToCode: { generate: props.generate },
+    });
+  }, [app, props.generate]);
+
+  return null;
+};

packages/excalidraw/components/LayerUI.tsx

@@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement";
 import { ShapeCache } from "../scene/ShapeCache";
 import Scene from "../scene/Scene";
 import { LaserPointerButton } from "./LaserPointerButton";
-import { MagicSettings } from "./MagicSettings";
 import { TTDDialog } from "./TTDDialog/TTDDialog";
 import { Stats } from "./Stats";
 import { actionToggleStats } from "../actions";
@@ -85,14 +84,6 @@ interface LayerUIProps {
   children?: React.ReactNode;
   app: AppClassProperties;
   isCollaborating: boolean;
-  openAIKey: string | null;
-  isOpenAIKeyPersisted: boolean;
-  onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void;
-  onMagicSettingsConfirm: (
-    apiKey: string,
-    shouldPersist: boolean,
-    source: "tool" | "generation" | "settings",
-  ) => void;
 }
 
 const DefaultMainMenu: React.FC<{
@@ -149,10 +140,6 @@ const LayerUI = ({
   children,
   app,
   isCollaborating,
-  openAIKey,
-  isOpenAIKeyPersisted,
-  onOpenAIAPIKeyChange,
-  onMagicSettingsConfirm,
 }: LayerUIProps) => {
   const device = useDevice();
   const tunnels = useInitializeTunnels();
@@ -482,25 +469,6 @@ const LayerUI = ({
           }}
         />
       )}
-      {appState.openDialog?.name === "settings" && (
-        <MagicSettings
-          openAIKey={openAIKey}
-          isPersisted={isOpenAIKeyPersisted}
-          onChange={onOpenAIAPIKeyChange}
-          onConfirm={(apiKey, shouldPersist) => {
-            const source =
-              appState.openDialog?.name === "settings"
-                ? appState.openDialog?.source
-                : "settings";
-            setAppState({ openDialog: null }, () => {
-              onMagicSettingsConfirm(apiKey, shouldPersist, source);
-            });
-          }}
-          onClose={() => {
-            setAppState({ openDialog: null });
-          }}
-        />
-      )}
       <ActiveConfirmDialog />
       <tunnels.OverwriteConfirmDialogTunnel.Out />
       {renderImageExportDialog()}

packages/excalidraw/components/MagicSettings.scss

@@ -1,18 +0,0 @@
-.excalidraw {
-  .MagicSettings {
-    .Island {
-      height: 100%;
-      display: flex;
-      flex-direction: column;
-    }
-  }
-
-  .MagicSettings-confirm {
-    padding: 0.5rem 1rem;
-  }
-
-  .MagicSettings__confirm {
-    margin-top: 2rem;
-    margin-right: auto;
-  }
-}

packages/excalidraw/components/MagicSettings.tsx

@@ -1,160 +0,0 @@
-import { useState } from "react";
-import { Dialog } from "./Dialog";
-import { TextField } from "./TextField";
-import { MagicIcon, OpenAIIcon } from "./icons";
-import { FilledButton } from "./FilledButton";
-import { CheckboxItem } from "./CheckboxItem";
-import { KEYS } from "../keys";
-import { useUIAppState } from "../context/ui-appState";
-import { InlineIcon } from "./InlineIcon";
-import { Paragraph } from "./Paragraph";
-
-import "./MagicSettings.scss";
-import TTDDialogTabs from "./TTDDialog/TTDDialogTabs";
-import { TTDDialogTab } from "./TTDDialog/TTDDialogTab";
-
-export const MagicSettings = (props: {
-  openAIKey: string | null;
-  isPersisted: boolean;
-  onChange: (key: string, shouldPersist: boolean) => void;
-  onConfirm: (key: string, shouldPersist: boolean) => void;
-  onClose: () => void;
-}) => {
-  const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || "");
-  const [shouldPersist, setShouldPersist] = useState<boolean>(
-    props.isPersisted,
-  );
-
-  const appState = useUIAppState();
-
-  const onConfirm = () => {
-    props.onConfirm(keyInputValue.trim(), shouldPersist);
-  };
-
-  if (appState.openDialog?.name !== "settings") {
-    return null;
-  }
-
-  return (
-    <Dialog
-      onCloseRequest={() => {
-        props.onClose();
-        props.onConfirm(keyInputValue.trim(), shouldPersist);
-      }}
-      title={
-        <div style={{ display: "flex" }}>
-          Wireframe to Code (AI){" "}
-          <div
-            style={{
-              display: "flex",
-              alignItems: "center",
-              justifyContent: "center",
-              padding: "0.1rem 0.5rem",
-              marginLeft: "1rem",
-              fontSize: 14,
-              borderRadius: "12px",
-              background: "var(--color-promo)",
-              color: "var(--color-surface-lowest)",
-            }}
-          >
-            Experimental
-          </div>
-        </div>
-      }
-      className="MagicSettings"
-      autofocus={false}
-    >
-      {/*  <h2
-        style={{
-          margin: 0,
-          fontSize: "1.25rem",
-          paddingLeft: "2.5rem",
-        }}
-      >
-        AI Settings
-      </h2> */}
-      <TTDDialogTabs dialog="settings" tab={appState.openDialog.tab}>
-        {/* <TTDDialogTabTriggers>
-          <TTDDialogTabTrigger tab="text-to-diagram">
-            <InlineIcon icon={brainIcon} /> Text to diagram
-          </TTDDialogTabTrigger>
-          <TTDDialogTabTrigger tab="diagram-to-code">
-            <InlineIcon icon={MagicIcon} /> Wireframe to code
-          </TTDDialogTabTrigger>
-        </TTDDialogTabTriggers> */}
-        {/* <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
-          TODO
-        </TTDDialogTab> */}
-        <TTDDialogTab
-          //  className="ttd-dialog-content"
-          tab="diagram-to-code"
-        >
-          <Paragraph>
-            For the diagram-to-code feature we use{" "}
-            <InlineIcon icon={OpenAIIcon} />
-            OpenAI.
-          </Paragraph>
-          <Paragraph>
-            While the OpenAI API is in beta, its use is strictly limited — as
-            such we require you use your own API key. You can create an{" "}
-            <a
-              href="https://platform.openai.com/login?launch"
-              rel="noopener noreferrer"
-              target="_blank"
-            >
-              OpenAI account
-            </a>
-            , add a small credit (5 USD minimum), and{" "}
-            <a
-              href="https://platform.openai.com/api-keys"
-              rel="noopener noreferrer"
-              target="_blank"
-            >
-              generate your own API key
-            </a>
-            .
-          </Paragraph>
-          <Paragraph>
-            Your OpenAI key does not leave the browser, and you can also set
-            your own limit in your OpenAI account dashboard if needed.
-          </Paragraph>
-          <TextField
-            isRedacted
-            value={keyInputValue}
-            placeholder="Paste your API key here"
-            label="OpenAI API key"
-            onChange={(value) => {
-              setKeyInputValue(value);
-              props.onChange(value.trim(), shouldPersist);
-            }}
-            selectOnRender
-            onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()}
-          />
-          <Paragraph>
-            By default, your API token is not persisted anywhere so you'll need
-            to insert it again after reload. But, you can persist locally in
-            your browser below.
-          </Paragraph>
-
-          <CheckboxItem checked={shouldPersist} onChange={setShouldPersist}>
-            Persist API key in browser storage
-          </CheckboxItem>
-
-          <Paragraph>
-            Once API key is set, you can use the <InlineIcon icon={MagicIcon} />{" "}
-            tool to wrap your elements in a frame that will then allow you to
-            turn it into code. This dialog can be accessed using the{" "}
-            <b>AI Settings</b> <InlineIcon icon={OpenAIIcon} />.
-          </Paragraph>
-
-          <FilledButton
-            className="MagicSettings__confirm"
-            size="large"
-            label="Confirm"
-            onClick={onConfirm}
-          />
-        </TTDDialogTab>
-      </TTDDialogTabs>
-    </Dialog>
-  );
-};

packages/excalidraw/components/TTDDialog/TTDDialogTabs.tsx

@@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils";
 const TTDDialogTabs = (
   props: {
     children: ReactNode;
-  } & (
+  } & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" },
-    | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
-    | { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
-  ),
 ) => {
   const setAppState = useExcalidrawSetAppState();
 
@@ -39,13 +36,6 @@ const TTDDialogTabs = (
           }
         }
         if (
-          props.dialog === "settings" &&
-          isMemberOf(["text-to-diagram", "diagram-to-code"], tab)
-        ) {
-          setAppState({
-            openDialog: { name: props.dialog, tab, source: "settings" },
-          });
-        } else if (
           props.dialog === "ttd" &&
           isMemberOf(["text-to-diagram", "mermaid"], tab)
         ) {

packages/excalidraw/components/dropdownMenu/DropdownMenuItemLink.tsx

@@ -13,6 +13,7 @@ const DropdownMenuItemLink = ({
   onSelect,
   className = "",
   selected,
+  rel = "noreferrer",
   ...rest
 }: {
   href: string;
@@ -22,6 +23,7 @@ const DropdownMenuItemLink = ({
   className?: string;
   selected?: boolean;
   onSelect?: (event: Event) => void;
+  rel?: string;
 } & React.AnchorHTMLAttributes<HTMLAnchorElement>) => {
   const handleClick = useHandleDropdownMenuItemClick(rest.onClick, onSelect);
 

packages/excalidraw/data/magic.ts

@@ -1,105 +0,0 @@
-import { THEME } from "../constants";
-import type { Theme } from "../element/types";
-import type { DataURL } from "../types";
-import type { OpenAIInput, OpenAIOutput } from "./ai/types";
-
-export type MagicCacheData =
-  | {
-      status: "pending";
-    }
-  | { status: "done"; html: string }
-  | {
-      status: "error";
-      message?: string;
-      code: "ERR_GENERATION_INTERRUPTED" | string;
-    };
-
-const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design.
-Your role is to transform low-fidelity wireframes into working front-end HTML code.
-
-YOU MUST FOLLOW FOLLOWING RULES:
-
-- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype
-- Leverage Tailwind for styling and layout (import as script <script src="https://cdn.tailwindcss.com"></script>)
-- Inline JavaScript when needed
-- Fetch dependencies from CDNs when needed (using unpkg or skypack)
-- Source images from Unsplash or create applicable placeholders
-- Interpret annotations as intended vs literal UI
-- Fill gaps using your expertise in UX and business logic
-- generate primarily for desktop UI, but make it responsive.
-- Use grid and flexbox wherever applicable.
-- Convert the wireframe in its entirety, don't omit elements if possible.
-
-If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification.
-
-Your goal is a production-ready prototype that brings the wireframes to life.
-
-Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`;
-
-export async function diagramToHTML({
-  image,
-  apiKey,
-  text,
-  theme = THEME.LIGHT,
-}: {
-  image: DataURL;
-  apiKey: string;
-  text: string;
-  theme?: Theme;
-}) {
-  const body: OpenAIInput.ChatCompletionCreateParamsBase = {
-    model: "gpt-4-vision-preview",
-    // 4096 are max output tokens allowed for `gpt-4-vision-preview` currently
-    max_tokens: 4096,
-    temperature: 0.1,
-    messages: [
-      {
-        role: "system",
-        content: SYSTEM_PROMPT,
-      },
-      {
-        role: "user",
-        content: [
-          {
-            type: "image_url",
-            image_url: {
-              url: image,
-              detail: "high",
-            },
-          },
-          {
-            type: "text",
-            text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`,
-          },
-          {
-            type: "text",
-            text,
-          },
-        ],
-      },
-    ],
-  };
-
-  let result:
-    | ({ ok: true } & OpenAIOutput.ChatCompletion)
-    | ({ ok: false } & OpenAIOutput.APIError);
-
-  const resp = await fetch("https://api.openai.com/v1/chat/completions", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify(body),
-  });
-
-  if (resp.ok) {
-    const json: OpenAIOutput.ChatCompletion = await resp.json();
-    result = { ...json, ok: true };
-  } else {
-    const json: OpenAIOutput.APIError = await resp.json();
-    result = { ...json, ok: false };
-  }
-
-  return result;
-}

packages/excalidraw/element/index.ts

@@ -46,7 +46,7 @@ export {
   dragNewElement,
 } from "./dragElements";
 export { isTextElement, isExcalidrawElement } from "./typeChecks";
-export { redrawTextBoundingBox } from "./textElement";
+export { redrawTextBoundingBox, getTextFromElements } from "./textElement";
 export {
   getPerfectElementSize,
   getLockedLinearCursorAlignSize,

packages/excalidraw/element/textElement.ts

@@ -886,3 +886,19 @@ export const getMinTextElementWidth = (
 ) => {
   return measureText("", font, lineHeight).width + BOUND_TEXT_PADDING * 2;
 };
+
+/** retrieves text from text elements and concatenates to a single string */
+export const getTextFromElements = (
+  elements: readonly ExcalidrawElement[],
+  separator = "\n\n",
+) => {
+  const text = elements
+    .reduce((acc: string[], element) => {
+      if (isTextElement(element)) {
+        acc.push(element.text);
+      }
+      return acc;
+    }, [])
+    .join(separator);
+  return text;
+};

packages/excalidraw/element/types.ts

@@ -12,7 +12,6 @@ import type {
   Merge,
   ValueOf,
 } from "../utility-types";
-import type { MagicCacheData } from "../data/magic";
 
 export type ChartType = "bar" | "line";
 export type FillStyle = "hachure" | "cross-hatch" | "solid" | "zigzag";
@@ -101,11 +100,22 @@ export type ExcalidrawEmbeddableElement = _ExcalidrawElementBase &
     type: "embeddable";
   }>;
 
+export type MagicGenerationData =
+  | {
+      status: "pending";
+    }
+  | { status: "done"; html: string }
+  | {
+      status: "error";
+      message?: string;
+      code: "ERR_GENERATION_INTERRUPTED" | string;
+    };
+
 export type ExcalidrawIframeElement = _ExcalidrawElementBase &
   Readonly<{
     type: "iframe";
     // TODO move later to AI-specific frame
-    customData?: { generationData?: MagicCacheData };
+    customData?: { generationData?: MagicGenerationData };
   }>;
 
 export type ExcalidrawIframeLikeElement =

packages/excalidraw/index.tsx

@@ -213,6 +213,7 @@ export {
   hashString,
   isInvisiblySmallElement,
   getNonDeletedElements,
+  getTextFromElements,
 } from "./element";
 export { defaultLang, useI18n, languages } from "./i18n";
 export {
@@ -287,3 +288,6 @@ export {
   isElementInsideBBox,
   elementPartiallyOverlapsWithOrContainsBBox,
 } from "../utils/withinBounds";
+
+export { DiagramToCodePlugin } from "./components/DiagramToCodePlugin/DiagramToCodePlugin";
+export { getDataURL } from "./data/blob";

packages/excalidraw/locales/en.json

@@ -272,8 +272,7 @@
     "laser": "Laser pointer",
     "hand": "Hand (panning tool)",
     "extraTools": "More tools",
-    "mermaidToExcalidraw": "Mermaid to Excalidraw",
+    "mermaidToExcalidraw": "Mermaid to Excalidraw"
-    "magicSettings": "AI settings"
   },
   "element": {
     "rectangle": "Rectangle",

packages/excalidraw/types.ts

@@ -325,14 +325,6 @@ export interface AppState {
   openDialog:
     | null
     | { name: "imageExport" | "help" | "jsonExport" }
-    | {
-        name: "settings";
-        source:
-          | "tool" // when magicframe tool is selected
-          | "generation" // when magicframe generate button is clicked
-          | "settings"; // when AI settings dialog is explicitly invoked
-        tab: "text-to-diagram" | "diagram-to-code";
-      }
     | { name: "ttd"; tab: "text-to-diagram" | "mermaid" }
     | { name: "commandPalette" };
   /**
@@ -655,6 +647,8 @@ export type AppClassProperties = {
   dismissLinearEditor: App["dismissLinearEditor"];
   flowChartCreator: App["flowChartCreator"];
   getEffectiveGridSize: App["getEffectiveGridSize"];
+  setPlugins: App["setPlugins"];
+  plugins: App["plugins"];
 };
 
 export type PointerDownState = Readonly<{
@@ -842,3 +836,8 @@ export type PendingExcalidrawElements = ExcalidrawElement[];
 export type NullableGridSize =
   | (AppState["gridSize"] & MakeBrand<"NullableGridSize">)
   | null;
+
+export type GenerateDiagramToCode = (props: {
+  frame: ExcalidrawMagicFrameElement;
+  children: readonly ExcalidrawElement[];
+}) => MaybePromise<{ html: string }>;

packages/excalidraw/utils.ts

@@ -1166,3 +1166,11 @@ export const promiseTry = async <TValue, TArgs extends unknown[]>(
 
 export const isAnyTrue = (...args: boolean[]): boolean =>
   Math.max(...args.map((arg) => (arg ? 1 : 0))) > 0;
+
+export const safelyParseJSON = (json: string): Record<string, any> | null => {
+  try {
+    return JSON.parse(json);
+  } catch {
+    return null;
+  }
+};