Implementation of CoreML + Live Render Preview

backends/swift_backend/LICENSE.md

@@ -0,0 +1,39 @@
+Copyright (C) 2022 Apple Inc. All Rights Reserved.
+
+IMPORTANT:  This Apple software is supplied to you by Apple
+Inc. ("Apple") in consideration of your agreement to the following
+terms, and your use, installation, modification or redistribution of
+this Apple software constitutes acceptance of these terms.  If you do
+not agree with these terms, please do not use, install, modify or
+redistribute this Apple software.
+
+In consideration of your agreement to abide by the following terms, and
+subject to these terms, Apple grants you a personal, non-exclusive
+license, under Apple's copyrights in this original Apple software (the
+"Apple Software"), to use, reproduce, modify and redistribute the Apple
+Software, with or without modifications, in source and/or binary forms;
+provided that if you redistribute the Apple Software in its entirety and
+without modifications, you must retain this notice and the following
+text and disclaimers in all such redistributions of the Apple Software.
+Neither the name, trademarks, service marks or logos of Apple Inc. may
+be used to endorse or promote products derived from the Apple Software
+without specific prior written permission from Apple.  Except as
+expressly stated in this notice, no other rights or licenses, express or
+implied, are granted by Apple herein, including but not limited to any
+patent rights that may be infringed by your derivative works or by other
+works in which the Apple Software may be incorporated.
+
+The Apple Software is provided by Apple on an "AS IS" basis.  APPLE
+MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION
+THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND
+OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS.
+
+IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION,
+MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED
+AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE),
+STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

backends/swift_backend/main.swift

@@ -0,0 +1,201 @@
+// For licensing see accompanying LICENSE.md file.
+// Copyright (C) 2022 Apple Inc. All Rights Reserved.
+
+import CoreML
+import StableDiffusion
+import UniformTypeIdentifiers
+
+import Darwin
+setbuf(stdout, nil)
+
+enum RunError: Error {
+    case resources(String)
+    case saving(String)
+}
+
+enum SchedulerOption: String {
+    case pndm, dpmpp
+    var stableDiffusionScheduler: StableDiffusionScheduler {
+        switch self {
+        case .pndm: return .pndmScheduler
+        case .dpmpp: return .dpmSolverMultistepScheduler
+        }
+    }
+}
+
+func log(_ str: String, term: String = "") {
+       print(str, terminator: term)
+}
+
+let fm = FileManager.default
+
+struct DiffusionBee {
+    var resourcePath: String = fm.homeDirectoryForCurrentUser.path+"/.diffusionbee/coreml_models/coreml-stable-diffusion-v1-5_split_einsum_compiled"
+    var outputPath: String = fm.homeDirectoryForCurrentUser.path+"/.diffusionbee/images"
+    var disableSafety: Bool = true
+    var reduceMemory: Bool = true
+
+    var prompt: String = ""
+    var negativePrompt: String = ""
+
+    var imageCount: Int = 1
+    var seed: UInt32 = 93
+    var saveEvery: Int = 0
+
+    func handleProgress(
+            _ progress: StableDiffusionPipeline.Progress,
+            _ sampleTimer: SampleTimer
+        ) { 
+            log("\u{1B}[1A\u{1B}[K")
+            log("Step \(progress.step) of \(progress.stepCount) ")
+            log(" [")
+            log(String(format: "mean: %.2f, ", 1.0/sampleTimer.mean))
+            log(String(format: "median: %.2f, ", 1.0/sampleTimer.median))
+            log(String(format: "last %.2f", 1.0/sampleTimer.allSamples.last!))
+            log("] step/sec")
+            // print("sdbk dnpr "+str(i) ) # done percentage
+           if saveEvery > 0, progress.step % saveEvery == 0 {
+               let saveCount = (try? saveImages(progress.currentImages, step: progress.step, logNames: true))?.count ?? 0
+               log(" saved \(saveCount) image\(saveCount != 1 ? "s" : "")")
+           }
+            log("\n")
+            let progressPercentage = Float(progress.step) / Float(progress.stepCount)
+            let progressPercentageInt = Int(ceil(progressPercentage * 100))
+            print("sdbk dnpr \(progressPercentageInt)")
+        }
+
+    func saveImages(
+            _ images: [CGImage?],
+            step: Int? = nil,
+            logNames: Bool = false
+    ) throws -> [String] {
+            let url = URL(filePath: outputPath)
+            var saved = [String]()
+            for i in 0 ..< images.count {
+
+                guard let image = images[i] else {
+                    if logNames {
+                        log("Image \(i) failed safety check and was not saved")
+                    }
+                    continue
+                }
+                let name = imageName(i, step: step)
+                let fileURL = url.appending(path:name)
+
+                guard let dest = CGImageDestinationCreateWithURL(fileURL as CFURL, UTType.png.identifier as CFString, 1, nil) else {
+                    throw RunError.saving("Failed to create destination for \(fileURL)")
+                }
+                CGImageDestinationAddImage(dest, image, nil)
+                if !CGImageDestinationFinalize(dest) {
+                    throw RunError.saving("Failed to save \(fileURL)")
+                }
+                if logNames {
+                    log("Saved \(name)\n")
+                    print("sdbk nwim \(fileURL.path)")
+                }
+                saved.append(fileURL.path)
+            }
+            return saved
+        }
+        func imageName(_ sample: Int, step: Int? = nil) -> String {
+        let fileCharLimit = 75
+            var name = "\(seed)"
+            name += prompt.prefix(fileCharLimit).replacingOccurrences(of: " ", with: "_")
+        if imageCount != 1 {
+            name += "_\(sample)"
+        }
+        name += ".png"
+        return name
+    }
+
+    mutating func run() throws {
+        guard FileManager.default.fileExists(atPath: resourcePath) else {
+        throw RunError.resources("Resource path does not exist \(resourcePath)")
+        }
+        let config = MLModelConfiguration()
+
+        print("sdbk mltl Loading Model")
+
+        let resourceURL = URL(filePath: resourcePath)
+        let computeUnits: MLComputeUnits = .cpuAndNeuralEngine
+
+        config.computeUnits = computeUnits
+
+        print("sdbk gnms  Loading SD Model"  )
+        let pipeline = try StableDiffusionPipeline(resourcesAt: resourceURL,
+                                                           configuration: config,
+                                                           disableSafety: disableSafety,
+                                                           reduceMemory: reduceMemory)
+        try pipeline.loadResources()
+        print("sdbk mdvr 1.5CoreML")
+
+
+        print("sdbk mlpr -1")
+        print("sdbk mdld")
+
+        while true {
+             print("sdbk inrd") // input ready
+             let input = readLine()!
+             if input == "" {
+                 continue
+             }
+
+             if !input.contains("b2s t2im") {
+                 continue
+             }
+
+             let inp_str = input.replacingOccurrences(of: "b2s t2im", with: "").trimmingCharacters(in: .whitespacesAndNewlines)
+
+            var d = ["W":512, "H":512, "num_imgs":1, "ddim_steps":25, "scale":7.5, "batch_size":1, "input_image":"", "img_strength":0.5, "negative_prompt":"", "mask_image":"", "model_id":0, "custom_model_path":"", "save_every": 0] as [String : Any]
+
+            let d_ = try JSONSerialization.jsonObject(with: inp_str.data(using: .utf8)!, options: []) as! [String:Any]
+            for (k,v) in d_ {
+                d[k] = v
+            }
+            print("sdbk inwk") // working on the input
+
+            print("Sampling ...\n")
+            let sampleTimer = SampleTimer()
+            sampleTimer.start()
+
+
+            prompt = d["prompt"] as! String
+            negativePrompt = d["negative_prompt"] as! String
+            imageCount = d["num_imgs"] as! Int
+            saveEvery = d["save_every"] as! Int
+            let stepCount = d["ddim_steps"] as! Int
+            let guidanceScale = d["scale"] as! Double
+
+            seed = d["seed"] as? UInt32 ?? UInt32.random(in: 0...UInt32.max)
+
+            var scheduler: SchedulerOption = .pndm
+
+            let images = try pipeline.generateImages(
+                       prompt: prompt,
+                       negativePrompt: negativePrompt,
+                       imageCount: imageCount,
+                       stepCount: stepCount,
+                       seed: seed,
+                       guidanceScale: Float(guidanceScale),
+                       scheduler: scheduler.stableDiffusionScheduler
+                   ) { progress in
+                       sampleTimer.stop()
+                       handleProgress(progress,sampleTimer)
+                       if progress.stepCount != progress.step {
+                           sampleTimer.start()
+                       }
+                       return true
+                   }
+            let _ = try saveImages(images, logNames: true)
+            print("sdbk igws")
+
+
+        }
+    }
+}
+if #available(iOS 16.2, macOS 13.1, *) {
+    var diffusionbee  = DiffusionBee()
+    try diffusionbee.run()
+} else {
+    print("Unsupported OS")
+}

backends/swift_backend/ml-stable-diffusion-main/.github/pull_request_template.md

@@ -0,0 +1,8 @@
+Thank you for your interest in contributing to Core ML Stable Diffusion! Please review [CONTRIBUTING.md](../CONTRIBUTING.md) first. If you would like to proceed with making a pull request, please indicate your agreement to the terms outlined in CONTRIBUTING.md by checking the box below. If not, please go ahead and fork this repo and make your updates.
+
+We appreciate your interest in the project!
+
+Do not erase the below when submitting your pull request:
+#########
+
+- [ ] I agree to the terms outlined in CONTRIBUTING.md 

backends/swift_backend/ml-stable-diffusion-main/.gitignore

@@ -0,0 +1,144 @@
+# Swift Package
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+.swiftpm
+.vscode
+.*.sw?
+*.docc-build
+*.vs
+Package.resolved
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# macOS filesystem
+*.DS_Store