Stable Diffusion inpaint cloth and background at the same mask

Sarit Ritwirune
3 min readJul 17, 2023

Starting point is https://www.facebook.com/groups/209358916852889/permalink/1046214066500699/

She would like to wear bikini and sit in the beach.

The original picture.

The original picture

Controlnet is https://huggingface.co/lllyasviel/control_v11p_sd15_inpaint

Model ID is https://civitai.com/models/6424/chilloutmix

Next one is masked image

Exclude human skin from A.I. working area.
import itertools
import math
import os
import random

# !pip install transformers accelerate
from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler
from diffusers.utils import load_image
import numpy as np
import torch
from tqdm import tqdm

from set_seed import seed_everything, resize_for_condition_image

seed: int = 8888
seed_everything(seed)
out_dir: str = "sasithorn_bikini_beach"
negative_prompt: str = "nsfw, ng_deepnegative_v1_75t,badhandv4, (worst quality:2), (low quality:2), (normal quality:2), lowres,watermark, monochrome"
device: str = "mps" if torch.backends.mps.is_available() else "cpu"
print(device)

# init_image = init_image.resize((512, 512))
strengths = [1,]
guidance_scales = [round(0.2 * _, 3) for _ in range(36, 81, 1)]
# eta_list = [round(0.2 * _, 3) for _ in range(0, 11, 1)]
prompt = "best quality, masterpiece, (photorealistic:1.4), 1girl, light smile, a girl sitting on the beach bed. She crosses her legs and wearing 2 piece string bikini"
combined_list = list(itertools.product(strengths, guidance_scales))
random.shuffle(combined_list)
init_image = load_image("sources/sasithorn.jpeg")
width, height = init_image.size
size_factor: float = 0.8
new_width, new_height = math.floor(width * size_factor / 8) * 8, math.floor(height * size_factor / 8) * 8
# new_width, new_height = width, height
print(width, height)
print(new_width, new_height)
init_image = init_image.resize((new_width, new_height))

generator = torch.Generator(device=device).manual_seed(seed)
mask_image = load_image("sources/masked_sasithorn.png").resize((new_width, new_height))



def make_inpaint_condition(image, image_mask):
image = np.array(image.convert("RGB")).astype(np.float32) / 255.0
image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0

assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size"
image[image_mask > 0.5] = 1 # set as masked pixel
image = np.expand_dims(image, 0).transpose(0, 3, 1, 2)
image = torch.from_numpy(image)
return image


def main() -> None:
"""Run main function."""
# strength = 1
# guidance_scale = 7.5
eta = 0
for item in tqdm(combined_list):
strength, guidance_scale = item
add_prompt = prompt
print(strength, guidance_scale, eta, add_prompt)
filename: str = f"{out_dir}/{add_prompt}_{strength}_{guidance_scale}_{eta}_0.png"
try:
if not os.path.exists(filename):
control_image = make_inpaint_condition(init_image, mask_image)

controlnet = ControlNetModel.from_pretrained(
"lllyasviel/control_v11p_sd15_inpaint",
).to(device)
pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
# "runwayml/stable-diffusion-v1-5", # Reference bad
# "../ai_directory/majicmixRealistic_v6", # Quite good
# "../ai_directory/MeinaV10", # Anime
# "../ai_directory/perfectWorld_v4Baked", # Ordinary
"../ai_directory/chilloutmix_NiPrunedFp32Fix", # It gives full bodysuit
controlnet=controlnet,
requires_safety_checker=False,
safety_checker=None
).to(device)
pipe.requires_safety_checker = False
pipe.safety_checker = None

pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)

# generate image
result = pipe(
prompt=add_prompt,
negative_prompt=negative_prompt,
num_inference_steps=100,
generator=generator,
image=init_image,
mask_image=mask_image,
control_image=control_image,
num_images_per_prompt=2,
width=new_width,
height=new_height,
strength=strength,
guidance_scale=guidance_scale,
# eta=eta,
)
for idx, image in enumerate(result.images):
filename: str = f"{out_dir}/{add_prompt}_{strength}_{guidance_scale}_{eta}_{idx}.png"
image.save(filename)
else:
print("File exists.")
except Exception as err:
print(err)
print(f"{filename} is impossible")
with open(f"{out_dir}/log.txt", "a") as file:
file.write(str(err) + "\n")
file.write(filename + "\n")
file.write("====================================" + "\n")

Results:

The outputs

Summary:

A.I. be able to replace cloth and background by single controlnet. The dirty pixels near legs come from rough cutting. They can be improved by redo masking again.

--

--

Sarit Ritwirune

On the way to full stack cross-platform. Currently make living by data science.