Zip Dependencies with Long Paths

.github/workflows/package-release.yml

@@ -45,6 +45,9 @@ jobs:
         shell: bash
         run: 'python -m pip install -r requirements/win-linux-cuda.txt --no-cache-dir --target .python_dependencies'
         working-directory: dream_textures
+      - name: Zip dependencies with long paths
+        shell: bash
+        run: 'python ./dream_textures/scripts/zip_dependencies.py'
       - name: Archive Release
         uses: thedoctor0/zip-release@main
         with:
@@ -73,6 +76,9 @@ jobs:
         shell: bash
         run: 'python -m pip install -r requirements/win-dml.txt --no-cache-dir --target .python_dependencies'
         working-directory: dream_textures
+      - name: Zip dependencies with long paths
+        shell: bash
+        run: 'python ./dream_textures/scripts/zip_dependencies.py'
       - name: Archive Release
         uses: thedoctor0/zip-release@main
         with:

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 *.so
 
 # Distribution / packaging
+.python_dependencies.zip
 .Python
 build/
 develop-eggs/

generator_process/actor.py

@@ -10,6 +10,19 @@
 from ..absolute_path import absolute_path
 from .future import Future
 
+def _patch_zip_direct_transformers_import():
+    # direct_transformers_import() implementation doesn't work when transformers is in a zip archive
+    # since it relies on existing file paths. The function appears to ensure the correct root module
+    # is obtained when there could be another loadable transformers module or it isn't in any sys.path
+    # directory during development testing, both not being a concern in this environment.
+    def direct_transformers_import(*_, **__):
+        import transformers
+        return transformers
+    from transformers.utils import import_utils
+    import_utils.direct_transformers_import = direct_transformers_import
+    from transformers import utils
+    utils.direct_transformers_import = direct_transformers_import
+
 def _load_dependencies():
     site.addsitedir(absolute_path(".python_dependencies"))
     deps = sys.path.pop(-1)
@@ -20,6 +33,9 @@ def _load_dependencies():
         python3_path = os.path.abspath(os.path.join(sys.executable, "..\\..\\..\\..\\python3.dll"))
         if os.path.exists(python3_path):
             os.add_dll_directory(os.path.dirname(python3_path))
+    if os.path.exists(absolute_path(".python_dependencies.zip")):
+        sys.path.insert(1, absolute_path(".python_dependencies.zip"))
+        _patch_zip_direct_transformers_import()
 
 main_thread_rendering = False
 is_actor_process = current_process().name == "__actor__"

scripts/zip_dependencies.py

@@ -0,0 +1,26 @@
+import shutil
+import zipfile
+from pathlib import Path
+
+
+def main():
+    root = Path(__file__).parent.parent
+    deps = root / '.python_dependencies'
+    deps_to_zip = [deps / 'transformers']
+
+    for dep in deps_to_zip:
+        if not dep.exists():
+            raise FileNotFoundError(dep)
+        elif not dep.is_dir():
+            raise EnvironmentError(f"not a directory {dep}")
+
+    zip_deps_path = root / '.python_dependencies.zip'
+    zip_deps_path.unlink(True)
+    with zipfile.PyZipFile(str(zip_deps_path), mode='x') as zip_deps:
+        for dep in deps_to_zip:
+            zip_deps.writepy(str(dep))
+            shutil.rmtree(str(dep))
+
+
+if __name__ == '__main__':
+    main()