RuntimeError: Input type (torch.cuda.HalfTensor) and weight type (torch.HalfTensor) should be the same

[HaroldHuanrongLIU]

Thanks for your great works!
I follow your introduction to build the environment, but the code didn't run.
My device: RTX 4090, CUDA 12.1, Python 3.8.

Epoch 0: | | 0/? [00:00<?, ?it/s]Traceback (most recent call last):
File "launch.py", line 239, in
main(args, extras)
File "launch.py", line 182, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 544, in fit
call._call_and_handle_interrupt(
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py", line 44, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 580, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 987, in _run
results = self._run_stage()
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1033, in _run_stage
self.fit_loop.run()
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 205, in run
self.advance()
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 363, in advance
self.epoch_loop.run(self._data_fetcher)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 140, in run
self.advance(data_fetcher)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 250, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 190, in run
self._optimizer_step(batch_idx, closure)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 268, in _optimizer_step
call._call_lightning_module_hook(
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
output = fn(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/core/module.py", line 1303, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py", line 152, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/plugins/precision/amp.py", line 80, in optimizer_step
closure_result = closure()
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 144, in call
self._result = self.closure(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 129, in closure
step_output = self._step_fn()
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 318, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
output = fn(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/pytorch_lightning/strategies/strategy.py", line 391, in training_step
return self.lightning_module.training_step(*args, **kwargs)
File "/home/huanrongliu/桌面/3DGS/HumanGaussian/threestudio/systems/GaussianDreamer.py", line 340, in training_step
guidance_out = self.guidance(
File "/home/huanrongliu/桌面/3DGS/HumanGaussian/threestudio/models/guidance/dual_branch_guidance.py", line 769, in call
latents = self.encode_images(rgb_BCHW_512.to(self.weights_dtype))
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/amp/autocast_mode.py", line 16, in decorate_autocast
return func(*args, **kwargs)
File "/home/huanrongliu/桌面/3DGS/HumanGaussian/threestudio/models/guidance/dual_branch_guidance.py", line 243, in encode_images
posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/diffusers/utils/accelerate_utils.py", line 46, in wrapper
return method(self, *args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/diffusers/models/autoencoders/autoencoder_kl.py", line 260, in encode
h = self.encoder(x)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/diffusers/models/autoencoders/vae.py", line 143, in forward
sample = self.conv_in(sample)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 460, in forward
return self._conv_forward(input, self.weight, self.bias)
File "/home/huanrongliu/miniconda3/envs/hg/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 456, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
RuntimeError: Input type (torch.cuda.HalfTensor) and weight type (torch.HalfTensor) should be the same

[HaroldHuanrongLIU]

btw, I used one 4090 24G.

[innocentARJOS]

the same problem, have you sloved it?

[silence-tang]

I encounter the same problem, how to fix it?

[innocentARJOS]

still trying

[silence-tang]

You could add self.pipe.to(self.device) after
self.pipe = StableDiffusionPipeline.from_pretrained( self.cfg.pretrained_model_name_or_path, unet=unet, vae=vae, torch_dtype=self.weights_dtype, ).to(torch_dtype=self.weights_dtype, torch_device=self.device)
which is in HumanGaussian/threestudio/models/guidance/dual_branch_guidance.py

[innocentARJOS]

it works out . Thanks!