feature: plugin to use "imaginAIry" web api for generating images locally

package.json

@@ -10,11 +10,13 @@
     "postinstall": "node -e \"try { require('husky').install() } catch (e) {if (e.code !== 'MODULE_NOT_FOUND') throw e}\" && yarn build",
     "stablestudio-plugin": "yarn workspace @stability/stablestudio-plugin",
     "stablestudio-plugin-example": "yarn workspace @stability/stablestudio-plugin-example",
+    "stablestudio-plugin-imaginairy": "yarn workspace @stability/stablestudio-plugin-imaginairy",
     "stablestudio-plugin-stability": "yarn workspace @stability/stablestudio-plugin-stability",
     "stablestudio-plugin-webgpu": "yarn workspace @stability/stablestudio-plugin-webgpu",
     "stablestudio-plugin-webui": "yarn workspace @stability/stablestudio-plugin-webui",
     "stablestudio-ui": "yarn workspace @stability/stablestudio-ui",
     "dev:use-example-plugin": "cross-env VITE_USE_EXAMPLE_PLUGIN=true yarn dev",
+    "dev:use-imaginairy-plugin": "cross-env VITE_USE_IMAGINAIRY_PLUGIN=true yarn dev",
     "dev": "yarn workspaces foreach --all --interlaced --verbose --parallel --jobs unlimited run dev",
     "build": "yarn workspaces foreach --all --interlaced --verbose --jobs unlimited run build",
     "clean": "yarn workspaces foreach --all --interlaced --verbose --parallel --jobs unlimited run clean && rimraf node_modules"

packages/stablestudio-plugin-imaginairy/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": ["../../.eslintrc.json"] }

packages/stablestudio-plugin-imaginairy/.gitignore

@@ -0,0 +1 @@
+lib

packages/stablestudio-plugin-imaginairy/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Bryce Drennan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/stablestudio-plugin-imaginairy/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@stability/stablestudio-plugin-imaginairy",
+  "version": "0.0.0",
+  "license": "MIT",
+  "main": "./lib/index.js",
+  "types": "./lib/index.d.ts",
+  "files": [
+    "lib"
+  ],
+  "scripts": {
+    "clean": "rimraf lib && rimraf node_modules",
+    "build:types": "ttsc --project tsconfig.json",
+    "build:javascript": "tsx scripts/Build.ts",
+    "build": "yarn build:types && yarn build:javascript",
+    "dev": "nodemon --watch src --ext ts,tsx,json --exec \"yarn build\""
+  },
+  "dependencies": {
+    "@stability/stablestudio-plugin": "workspace:^"
+  },
+  "devDependencies": {
+    "@typescript-eslint/eslint-plugin": "^5.33.1",
+    "@typescript-eslint/parser": "^5.33.1",
+    "eslint": "8.22.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-markdown": "^3.0.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-react": "^7.30.1",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "nodemon": "^2.0.20",
+    "prettier": "^2.7.1",
+    "rimraf": "^3.0.2",
+    "ts-node": "^10.9.1",
+    "tsx": "^3.12.1",
+    "ttypescript": "^1.5.13",
+    "typescript": "4.8.4",
+    "typescript-transform-paths": "^3.4.4"
+  }
+}

packages/stablestudio-plugin-imaginairy/scripts/Build.ts

@@ -0,0 +1,21 @@
+import * as ESBuild from "esbuild";
+
+const main = async () => {
+  try {
+    await ESBuild.build({
+      entryPoints: ["src/index.ts"],
+      outdir: "lib",
+      bundle: true,
+      sourcemap: true,
+      minify: true,
+      splitting: true,
+      format: "esm",
+      target: ["esnext"],
+    });
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+};
+
+main();

packages/stablestudio-plugin-imaginairy/src/index.ts

@@ -0,0 +1,156 @@
+import * as StableStudio from "@stability/stablestudio-plugin";
+
+const defaultApiUrl = "http://127.0.0.1:8000/api/stablestudio";
+
+export const createPlugin = StableStudio.createPlugin<{
+  imagesGeneratedSoFar: number;
+  settings: {
+    apiUrl: StableStudio.PluginSettingString;
+  };
+}>(({ set, get }) => ({
+  imagesGeneratedSoFar: 0,
+
+  manifest: {
+    name: "imaginAIry Local Diffusion Plugin",
+    author: "Bryce Drennan",
+    link: "https://github.com/brycedrennan/imaginAIry",
+    icon: `${window.location.origin}/DummyImage.png`,
+    version: "0.0.1",
+    license: "MIT",
+    description: "Generate images using imaginAIry.",
+  },
+
+  createStableDiffusionImages: async (options) => {
+    console.log(options);
+    // const image = await fetch(`${window.location.origin}/DummyImage.png`);
+
+    set(({ imagesGeneratedSoFar }) => ({
+      imagesGeneratedSoFar: imagesGeneratedSoFar + 4,
+    }));
+
+    const apiUrl = get().settings.apiUrl.value ?? defaultApiUrl;
+
+    const response = await fetch(apiUrl + "/generate", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: await jsonifyOptions(options),
+    });
+    const json = await response.json();
+    console.log(json);
+
+    type Image = {
+      id: string;
+      createdAt: string;
+      blob: string; // Or Blob if it's a binary blob
+    };
+
+    // Process the response from the server
+    const images = json.images.map((image: Image) => {
+      const blob = base64ToBlob(image.blob, "image/jpeg");
+
+      return {
+        id: image.id,
+        createdAt: new Date(image.createdAt),
+        blob: blob,
+      };
+    });
+
+    return {
+      id: `${Math.random() * 10000000}`,
+      images: images,
+    };
+  },
+  getStableDiffusionModels: async () => {
+    const apiUrl = get().settings.apiUrl.value ?? defaultApiUrl;
+    const response = await fetch(apiUrl + "/models");
+    return await response.json();
+  },
+  getStableDiffusionSamplers: async () => {
+    const apiUrl = get().settings.apiUrl.value ?? defaultApiUrl;
+    const response = await fetch(apiUrl + "/samplers");
+    return await response.json();
+  },
+  getStableDiffusionDefaultCount: () => 1,
+  getStableDiffusionDefaultInput: () => {
+    console.log("getStableDiffusionDefaultInput");
+    return {
+      steps: 16,
+      sampler: {
+        id: "k_dpmpp_2m",
+        name: "k_dpmpp_2m",
+      },
+      model: "SD-1.5",
+    };
+  },
+  getStatus: () => {
+    const { imagesGeneratedSoFar } = get();
+    return {
+      indicator: "success",
+      text:
+        imagesGeneratedSoFar > 0
+          ? `${imagesGeneratedSoFar} images generated`
+          : "Ready",
+    };
+  },
+
+  settings: {
+    apiUrl: {
+      type: "string" as const,
+      default: "",
+      placeholder: "URL to imaginAIry API",
+      value: localStorage.getItem("imaginairy-apiUrl") ?? defaultApiUrl,
+    },
+  },
+
+  setSetting: (key, value) => {
+    set(({ settings }) => ({
+      settings: {
+        [key]: { ...settings[key], value: value as string },
+      },
+    }));
+    const settingName = "imaginairy-" + key;
+    console.log(settingName + " : " + value);
+    localStorage.setItem(settingName, value as string);
+  },
+}));
+
+function base64ToBlob(base64: string, contentType = ""): Blob {
+  const byteCharacters = atob(base64);
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+  const byteArray = new Uint8Array(byteNumbers);
+  return new Blob([byteArray], { type: contentType });
+}
+
+function blobToBase64(blob: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      const base64Data = reader.result as string;
+      resolve(base64Data.split(",")[1]);
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(blob);
+  });
+}
+
+async function jsonifyOptions(options: any): Promise<string> {
+  // Deep copy of options
+  const copiedOptions = JSON.parse(JSON.stringify(options));
+
+  if (options?.input?.initialImage?.blob) {
+    const initImgB64 = await blobToBase64(options.input.initialImage.blob);
+    copiedOptions.input.initialImage.blob = initImgB64;
+  }
+
+  if (options?.input?.maskImage?.blob) {
+    const maskImgB64 = await blobToBase64(options.input.maskImage.blob);
+    copiedOptions.input.maskImage.blob = maskImgB64;
+  }
+
+  return JSON.stringify(copiedOptions);
+}

packages/stablestudio-plugin-imaginairy/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"],
+  "compilerOptions": {
+    "emitDeclarationOnly": true,
+    "declaration": true,
+    "noUncheckedIndexedAccess": false,
+
+    "outDir": "./lib",
+    "baseUrl": "./",
+    "paths": {
+      "~/*": ["./src/*"]
+    },
+
+    "plugins": [
+      { "transform": "typescript-transform-paths" },
+      { "transform": "typescript-transform-paths", "afterDeclarations": true }
+    ]
+  }
+}

packages/stablestudio-ui/src/Environment/index.tsx

@@ -8,6 +8,7 @@ declare global {
   interface ImportMetaEnv {
     readonly VITE_GIT_HASH: string;
     readonly VITE_USE_EXAMPLE_PLUGIN: string;
+    readonly VITE_USE_IMAGINAIRY_PLUGIN: string;
   }
 }
 
@@ -20,6 +21,7 @@ export namespace Environment {
   const variables = {
     VITE_GIT_HASH: import.meta.env.VITE_GIT_HASH,
     VITE_USE_EXAMPLE_PLUGIN: import.meta.env.VITE_USE_EXAMPLE_PLUGIN ?? "false",
+    VITE_USE_IMAGINAIRY_PLUGIN: import.meta.env.VITE_USE_IMAGINAIRY_PLUGIN ?? "false",
   } as const;
 
   export function get(name: VariableName): string {

packages/stablestudio-ui/src/Plugin/index.tsx

@@ -1,5 +1,6 @@
 import * as StableStudio from "@stability/stablestudio-plugin";
 import * as StableStudioPluginExample from "@stability/stablestudio-plugin-example";
+import * as StableStudioPluginImaginairy from "@stability/stablestudio-plugin-imaginairy";
 import * as StableStudioPluginStability from "@stability/stablestudio-plugin-stability";
 
 import { Environment } from "~/Environment";
@@ -115,6 +116,8 @@ namespace State {
     const { createPlugin: createRootPlugin } =
       Environment.get("USE_EXAMPLE_PLUGIN") === "true"
         ? StableStudioPluginExample
+        : Environment.get("USE_IMAGINAIRY_PLUGIN") === "true"
+        ? StableStudioPluginImaginairy
         : StableStudioPluginStability;
 
     return {

yarn.lock

@@ -1156,6 +1156,31 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@stability/stablestudio-plugin-imaginairy@workspace:packages/stablestudio-plugin-imaginairy":
+  version: 0.0.0-use.local
+  resolution: "@stability/stablestudio-plugin-imaginairy@workspace:packages/stablestudio-plugin-imaginairy"
+  dependencies:
+    "@stability/stablestudio-plugin": "workspace:^"
+    "@typescript-eslint/eslint-plugin": ^5.33.1
+    "@typescript-eslint/parser": ^5.33.1
+    eslint: 8.22.0
+    eslint-config-prettier: ^8.5.0
+    eslint-plugin-import: ^2.26.0
+    eslint-plugin-markdown: ^3.0.0
+    eslint-plugin-prettier: ^4.2.1
+    eslint-plugin-react: ^7.30.1
+    eslint-plugin-react-hooks: ^4.6.0
+    nodemon: ^2.0.20
+    prettier: ^2.7.1
+    rimraf: ^3.0.2
+    ts-node: ^10.9.1
+    tsx: ^3.12.1
+    ttypescript: ^1.5.13
+    typescript: 4.8.4
+    typescript-transform-paths: ^3.4.4
+  languageName: unknown
+  linkType: soft
+
 "@stability/stablestudio-plugin-stability@workspace:^, @stability/stablestudio-plugin-stability@workspace:packages/stablestudio-plugin-stability":
   version: 0.0.0-use.local
   resolution: "@stability/stablestudio-plugin-stability@workspace:packages/stablestudio-plugin-stability"