Stable Diffusion Cheatsheet: Pipelines, ControlNet, LoRA and img2img

# Core — the Hugging Face Diffusers library
pip install -U diffusers transformers accelerate safetensors

# Speedups — pick what your GPU supports
pip install xformers          # memory-efficient attention (CUDA)
pip install bitsandbytes      # 4/8-bit quantization for big checkpoints

# Adapters / utilities
pip install peft              # LoRA / IP-Adapter helpers
pip install controlnet-aux    # canny / depth / openpose preprocessors
pip install invisible-watermark

# Auth — many SDXL / Flux checkpoints are gated
pip install huggingface_hub
hf auth login                 # paste token from huggingface.co/settings/tokens

import torch
from diffusers import StableDiffusionXLPipeline, DPMSolverMultistepScheduler

pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    torch_dtype=torch.float16,
    variant="fp16",
    use_safetensors=True,
).to("cuda")

# DPM++ 2M Karras — fast, low step count, looks great on SDXL
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
    pipe.scheduler.config, use_karras_sigmas=True,
)
pipe.enable_xformers_memory_efficient_attention()

generator = torch.Generator(device="cuda").manual_seed(42)

image = pipe(
    prompt="cinematic photo of an old fisherman repairing nets at dawn, soft fog, 35mm",
    negative_prompt="cartoon, text, watermark, lowres",
    num_inference_steps=25,
    guidance_scale=7.0,
    width=1024, height=1024,
    generator=generator,
).images[0]

image.save("fisherman.png")

import torch
from diffusers import StableDiffusionXLImg2ImgPipeline
from diffusers.utils import load_image

pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-refiner-1.0",
    torch_dtype=torch.float16,
    variant="fp16",
).to("cuda")

init = load_image("sketch.png").resize((1024, 1024))

# strength = how much to ignore the input (0 = identity, 1 = pure txt2img)
result = pipe(
    prompt="oil painting of the same scene, brush texture, warm tones",
    image=init,
    strength=0.55,
    num_inference_steps=30,
    guidance_scale=7.5,
).images[0]

result.save("painting.png")

import torch
from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel
from diffusers.utils import load_image
from controlnet_aux import CannyDetector

# 1 · Preprocess the conditioning image into a Canny edge map
canny = CannyDetector()
control = canny(load_image("photo.jpg"), low_threshold=100, high_threshold=200)

# 2 · Load matching ControlNet for SDXL
controlnet = ControlNetModel.from_pretrained(
    "diffusers/controlnet-canny-sdxl-1.0",
    torch_dtype=torch.float16,
)

pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    controlnet=controlnet,
    torch_dtype=torch.float16,
).to("cuda")
pipe.enable_xformers_memory_efficient_attention()

image = pipe(
    prompt="a sun-bleached temple at golden hour, ultra-detailed",
    image=control,
    controlnet_conditioning_scale=0.8,
    num_inference_steps=30,
    guidance_scale=6.5,
).images[0]
image.save("temple.png")

import torch
from diffusers import StableDiffusionXLPipeline, EulerAncestralDiscreteScheduler

pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    torch_dtype=torch.float16,
    variant="fp16",
).to("cuda")
pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
pipe.enable_xformers_memory_efficient_attention()

# Load two LoRAs from the Hub and assign weights
pipe.load_lora_weights("nerijs/pixel-art-xl", adapter_name="pixel")
pipe.load_lora_weights("CiroN2022/cyberpunk-style", adapter_name="cyber")
pipe.set_adapters(["pixel", "cyber"], adapter_weights=[0.7, 0.4])

# Memory savers — useful when you stack adapters
pipe.enable_model_cpu_offload()        # weights to CPU between calls
pipe.enable_vae_tiling()               # tiled VAE for high-res

gen = torch.Generator(device="cuda").manual_seed(7)

image = pipe(
    prompt="pixel art, cyberpunk neon market street, rain, vending machines",
    negative_prompt="blurry, jpeg, photoreal, text",
    num_inference_steps=28,
    guidance_scale=7.0,
    generator=gen,
).images[0]

image.save("market.png")
pipe.unload_lora_weights()             # release adapter memory