Torch Compile Error: torch._dynamo.exc.Unsupported: hasattr: TensorVariable()

[dblunk88]

Describe the bug

Getting this error:

  0%|          | 0/100 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "/home/david/PycharmProjects/local_diffusion/deepfloyd.py", line 61, in <module>
    image = pipe(
  File "/home/david/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/home/david/PycharmProjects/local_diffusion/diffusers/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py", line 791, in __call__
    noise_pred = self.unet(
  File "/home/david/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 280, in _fn
    return fn(*args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 433, in catch_errors
    return callback(frame, cache_size, hooks, frame_state)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 122, in _fn
    return fn(*args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 337, in _convert_frame_assert
    return _compile(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
    r = func(*args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 407, in _compile
    out_code = transform_code_object(code, transform)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/bytecode_transformation.py", line 1000, in transform_code_object
    transformations(instructions, code_options)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py", line 392, in transform
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2006, in run
    super().run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1135, in CALL_FUNCTION_EX
    self.call_function(fn, argsvars.items, kwargsvars.items)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 306, in call_function
    return super().call_function(tx, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 269, in call_function
    return super().call_function(tx, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1095, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 269, in call_function
    return super().call_function(tx, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1147, in CALL_FUNCTION_KW
    self.call_function(fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 269, in call_function
    return super().call_function(tx, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1095, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1135, in CALL_FUNCTION_EX
    self.call_function(fn, argsvars.items, kwargsvars.items)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 269, in call_function
    return super().call_function(tx, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1095, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/functions.py", line 102, in call_function
    return tx.inline_user_function_return(
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 590, in inline_user_function_return
    result = InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2111, in inline_call
    return cls.inline_call_(parent, func, args, kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 2189, in inline_call_
    tracer.run()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
    and self.step()
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
    getattr(self, inst.opname)(inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 385, in wrapper
    return inner_fn(self, inst)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 1095, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/symbolic_convert.py", line 554, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/builtin.py", line 584, in call_function
    result = handler(tx, *args, **kwargs)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/builtin.py", line 941, in call_hasattr
    return obj.call_hasattr(tx, name).add_options(self, obj, attr)
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/variables/base.py", line 225, in call_hasattr
    unimplemented(f"hasattr: {repr(self)}")
  File "/home/david/.local/lib/python3.10/site-packages/torch/_dynamo/exc.py", line 133, in unimplemented
    raise Unsupported(msg)
torch._dynamo.exc.Unsupported: hasattr: TensorVariable()

from user code:
   File "/home/david/.local/lib/python3.10/site-packages/accelerate/utils/operations.py", line 100, in recursively_apply
    elif test_type(data):
  File "/home/david/.local/lib/python3.10/site-packages/accelerate/utils/operations.py", line 131, in _has_to_method
    return hasattr(t, "to")

Set torch._dynamo.config.verbose=True or TORCHDYNAMO_VERBOSE=1 for more information


You can suppress this exception and fall back to eager by setting:
    import torch._dynamo
    torch._dynamo.config.suppress_errors = True

Reproduction

using the same code as in #3313

Logs

No response

System Info

newest diffusers, Ubuntu 22.04, Python3.10, nightly version of pytorch

[dblunk88]

updated accelerate to the latest github version, now I get:

Downloading shards: 100%|██████████| 2/2 [00:00<00:00, 5660.33it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00, 3.21s/it]
Traceback (most recent call last):
File "/home/david/PycharmProjects/local_diffusion/deepfloyd.py", line 28, in
pipe = DiffusionPipeline.from_pretrained(
File "/home/david/PycharmProjects/local_diffusion/diffusers/src/diffusers/pipelines/pipeline_utils.py", line 1039, in from_pretrained
loaded_sub_model = load_sub_model(
File "/home/david/PycharmProjects/local_diffusion/diffusers/src/diffusers/pipelines/pipeline_utils.py", line 445, in load_sub_model
loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
File "/home/david/PycharmProjects/local_diffusion/diffusers/src/diffusers/models/modeling_utils.py", line 616, in from_pretrained
accelerate.load_checkpoint_and_dispatch(model, model_file, device_map, dtype=torch_dtype)
File "/home/david/.local/lib/python3.10/site-packages/accelerate/big_modeling.py", line 466, in load_checkpoint_and_dispatch
max_memory = get_balanced_memory(
File "/home/david/.local/lib/python3.10/site-packages/accelerate/utils/modeling.py", line 526, in get_balanced_memory
mean_leaves = int(sum([module_sizes[n] for n in leaves]) / len(leaves))
ZeroDivisionError: division by zero

[dblunk88]

update: problem seems to be with device_map="auto"

[patrickvonplaten]

@dblunk88,

Could you please copy-paste an minimal, easily-reproducible code snippet?
It's pretty difficult to reproduce the same bug here. Thanks!

[Skquark]

I've also been getting the divide by zero error trying to implement DeepFloyd. The code is same as low-mem DeepFloyd Colab example, but seems to be very finicky with the versions of packages installed, and still can't get past with all the same pinned ~= versions like the example. Depending on the versions installed, was also getting errors related to the T5Tokenizer and name 'safe_open' is not defined, but managed to get past most of that by using older versions and changing install orders within my app. Is there a workaround by setting device_map to something else? I've been messing with it for over a week..

Update:
Got past that setting device_map=None on DiffusionPipeline there, so it looked like this:
text_encoder = T5EncoderModel.from_pretrained(model_id, subfolder="text_encoder", device_map="auto", load_in_8bit=True, variant="8bit")
pipe_deepfloyd = DiffusionPipeline.from_pretrained(model_id, text_encoder=text_encoder, unet=None, device_map=None)

But then I got the same div by zero error on the next stage of IFPipeline that also had "auto", however when I changed that to None too it gave a different error about using half. Something's not quite right...

[dblunk88]

I've also been getting the divide by zero error trying to implement DeepFloyd. The code is same as low-mem DeepFloyd Colab example, but seems to be very finicky with the versions of packages installed, and still can't get past with all the same pinned ~= versions like the example. Depending on the versions installed, was also getting errors related to the T5Tokenizer and name 'safe_open' is not defined, but managed to get past most of that by using older versions and changing install orders within my app. Is there a workaround by setting device_map to something else? I've been messing with it for over a week..

Update: Got past that setting device_map=None on DiffusionPipeline there, so it looked like this: text_encoder = T5EncoderModel.from_pretrained(model_id, subfolder="text_encoder", device_map="auto", load_in_8bit=True, variant="8bit") pipe_deepfloyd = DiffusionPipeline.from_pretrained(model_id, text_encoder=text_encoder, unet=None, device_map=None)

But then I got the same div by zero error on the next stage of IFPipeline that also had "auto", however when I changed that to None too it gave a different error about using half. Something's not quite right...

if you're using fp16, try removing that

[DarioSucic]

@patrickvonplaten

I could sidestep the accelerate loading issue by passing in device_map="sequential", but otherwise I'm getting the issue in the OP.

Env info:

accelerate==0.19.0
torch==2.1.0.dev20230515+cu121
diffusers @ git+https://github.com/huggingface/diffusers@29b1325a5ae28fa8d7f459b372582287ffc571e5
transformers==4.28.0

Below is a small example that crashes for me, and here's the associated stacktrace.

import torch
from diffusers import IFPipeline

pipe = IFPipeline.from_pretrained(
    "DeepFloyd/IF-I-XL-v1.0",
    text_encoder=None,
    device_map="sequential",
    torch_dtype=torch.float16,
    variant="fp16"
)

pipe.unet.to(memory_format=torch.channels_last)
pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)

prompt_embeds = torch.randn((1, 2, 4096), dtype=torch.float16)
neg_prompt_embeds = torch.randn((1, 2, 4096), dtype=torch.float16)

out = pipe(
    prompt_embeds=prompt_embeds,
    negative_prompt_embeds=neg_prompt_embeds,
    output_type="pt"
)

[DarioSucic]

As mentioned in the stacktrace the issue is related to a particular accelerate hook calling hasattr, and I can confirm that compiling works fine if the model is loaded without accelerate (i.e by not passing device_map), and then manually moved to a gpu.

Another workaround is to remove accelerate hooks by calling pipe.remove_all_hooks(), though I think that could break multi-gpu and/or cpu offloading?

[patrickvonplaten]

Interesting! To better understand, why do you pass device_map="sequential",?
Is this really necessary?

The following should work no:

import torch
from diffusers import IFPipeline

pipe = IFPipeline.from_pretrained(
    "DeepFloyd/IF-I-XL-v1.0",
    text_encoder=None,
    torch_dtype=torch.float16,
    variant="fp16"
)
pipe.enable_model_cpu_offload()

pipe.unet.to(memory_format=torch.channels_last)
pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)

prompt_embeds = torch.randn((1, 2, 4096), dtype=torch.float16)
neg_prompt_embeds = torch.randn((1, 2, 4096), dtype=torch.float16)

out = pipe(
    prompt_embeds=prompt_embeds,
    negative_prompt_embeds=neg_prompt_embeds,
    output_type="pt"
)

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the contributing guidelines are likely to be ignored.