ControlNet và Style Transfer

1from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel
2from diffusers.utils import load_image
3import cv2
4import numpy as np
5from PIL import Image
6
7# Load ControlNet
8controlnet = ControlNetModel.from_pretrained(
9    "diffusers/controlnet-canny-sdxl-1.0",
10    torch_dtype=torch.float16
11)
12
13pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
14    "stabilityai/stable-diffusion-xl-base-1.0",
15    controlnet=controlnet,
16    torch_dtype=torch.float16
17).to("cuda")
18
19# Extract edges
20image = np.array(Image.open("input.png"))
21edges = cv2.Canny(image, 100, 200)
22edge_image = Image.fromarray(edges)
23
24# Generate voi edge control
25result = pipe(
26    prompt="beautiful watercolor painting, vibrant colors",
27    image=edge_image,
28    controlnet_conditioning_scale=0.7,
29    num_inference_steps=30
30).images[0]

1from transformers import pipeline
2
3# Extract depth map
4depth_estimator = pipeline("depth-estimation", model="Intel/dcs-large")
5depth = depth_estimator(Image.open("photo.png"))
6depth_image = depth["depth"]
7
8# Use depth for controlled generation
9controlnet_depth = ControlNetModel.from_pretrained(
10    "diffusers/controlnet-depth-sdxl-1.0",
11    torch_dtype=torch.float16
12)
13
14pipe_depth = StableDiffusionXLControlNetPipeline.from_pretrained(
15    "stabilityai/stable-diffusion-xl-base-1.0",
16    controlnet=controlnet_depth,
17    torch_dtype=torch.float16
18).to("cuda")
19
20result = pipe_depth(
21    prompt="anime style room, detailed, colorful",
22    image=depth_image,
23    controlnet_conditioning_scale=0.8
24).images[0]

1from controlnet_aux import OpenposeDetector
2
3# Detect pose
4pose_detector = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
5pose_image = pose_detector(Image.open("person.png"))
6
7# Generate voi pose control
8result = pipe_pose(
9    prompt="professional dancer in elegant dress, studio lighting",
10    image=pose_image,
11    controlnet_conditioning_scale=0.8
12).images[0]

1from diffusers import StableDiffusionXLPipeline
2
3pipe = StableDiffusionXLPipeline.from_pretrained(
4    "stabilityai/stable-diffusion-xl-base-1.0",
5    torch_dtype=torch.float16
6).to("cuda")
7
8# Load IP-Adapter
9pipe.load_ip_adapter(
10    "h94/IP-Adapter",
11    subfolder="sdxl_models",
12    weight_name="ip-adapter_sdxl.bin"
13)
14
15style_image = Image.open("style_reference.png")
16
17result = pipe(
18    prompt="a cozy cafe interior",
19    ip_adapter_image=style_image,
20    num_inference_steps=30
21).images[0]

1import torch
2import torchvision.transforms as transforms
3from torchvision.models import vgg19
4
5# Classic neural style transfer
6def style_transfer(content_img, style_img, steps=300):
7    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
8    
9    transform = transforms.Compose([
10        transforms.Resize(512),
11        transforms.ToTensor()
12    ])
13    
14    content = transform(content_img).unsqueeze(0).to(device)
15    style = transform(style_img).unsqueeze(0).to(device)
16    
17    # Use VGG19 features
18    vgg = vgg19(pretrained=True).features.to(device).eval()
19    
20    # Optimize target image
21    target = content.clone().requires_grad_(True)
22    optimizer = torch.optim.Adam([target], lr=0.01)
23    
24    for step in range(steps):
25        # Extract features and compute style + content loss
26        optimizer.zero_grad()
27        loss = compute_style_content_loss(vgg, target, content, style)
28        loss.backward()
29        optimizer.step()
30    
31    return target

1from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel
2
3# Combine multiple controls
4controlnets = [
5    ControlNetModel.from_pretrained("controlnet-canny", torch_dtype=torch.float16),
6    ControlNetModel.from_pretrained("controlnet-depth", torch_dtype=torch.float16),
7]
8
9pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
10    "stabilityai/stable-diffusion-xl-base-1.0",
11    controlnet=controlnets,
12    torch_dtype=torch.float16
13).to("cuda")
14
15result = pipe(
16    prompt="detailed architectural rendering",
17    image=[canny_image, depth_image],
18    controlnet_conditioning_scale=[0.7, 0.5]
19).images[0]