Real to Anime style with stable diffusion

Sarit Ritwirune
3 min readJul 31, 2023

Given input image is subset at the sea.

Sunset at the sea

In my full code it has many combinations, but my hardware is limited only M2 96GB MacBook Pro 2022. Then I turn off and do a combination only base_prompts, models

At the time being lora is not support mps yet. So I decided to use only checkpoint

import itertools
import typing as typ
import math
import os.path
import random
import cv2
import diffusers
import torch
from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
from diffusers.utils import load_image
from tqdm import tqdm
from PIL import Image
import numpy as np

from read_lora import load_lora_weights_orig
from set_seed import seed_everything

seed: int = 111
device = "mps" if torch.backends.mps.is_available() else "cpu"
moaw_dir: str = "/Users/sarit/study/try_openai/try_civitai/sources/moaw"
out_dir: str = "real2anime"

def get_canny_image(filename: str) -> Image:
"""Return canny image."""
image = load_image(filename)
image = np.array(image)
low_threshold = 100
high_threshold = 200
image = cv2.Canny(image, low_threshold, high_threshold)
image = image[:, :, None]
image = np.concatenate([image, image, image], axis=2)
canny_image = Image.fromarray(image)
return canny_image

base_prompts: typ.List[str] = [
"oil paint",
"water colour picture",

pictures = [
# ("the tables and chais in the cafe. the wall has fireplace over fireplace is tv", f"{moaw_dir}/9.jpg"),
# ("inside cafe. through glass door and glass window", f"{moaw_dir}/46350.jpg"),
# ("cafe with many tables and chairs. Wall is decorated with pictures", f"{moaw_dir}/46351.jpg"),
("a sunset at the sea", f"{moaw_dir}/46813.jpg"),
models = [
("abased_v10", "../ai_directory/abased_v10"),
# ("animeArtDiffusionXL_alpha3", "../ai_directory/animeArtDiffusionXL_alpha3"), Can't user contorlnet
("animePastelDream", "../ai_directory/animePastelDream"),
("anythingV3_fp16", "../ai_directory/anythingV3_fp16"),
("chilloutmix_NiPrunedFp32Fix", "../ai_directory/chilloutmix_NiPrunedFp32Fix"),
("exquisiteDetails_v10", "../ai_directory/exquisiteDetails_v10"),
("flat2DAnimerge", "../ai_directory/flat2DAnimerge"),
("henmixrealV10_henmixrealV10", "../ai_directory/henmixrealV10_henmixrealV10"),
("kawaiiRealisticAnime_a03", "../ai_directory/kawaiiRealisticAnime_a03"),
("lyricalVivian_v10", "../ai_directory/lyricalVivian_v10"),
("majicmixRealistic_v6", "../ai_directory/majicmixRealistic_v6"),
("MeinaV10", "../ai_directory/MeinaV10"),
("perfectWorld_v4Baked", "../ai_directory/perfectWorld_v4Baked"),
("realdosmix", "../ai_directory/realdosmix"),
("realisticVisionV40_v40VAE", "../ai_directory/realisticVisionV40_v40VAE"),
("realisticVisionV50_v50VAE", "../ai_directory/realisticVisionV50_v50VAE"),
# ("sd_xl_base_0.9", "../ai_directory/sd_xl_base_0.9"), # Can't use controlnet
# schedulers = [
# ("LMSDiscreteScheduler", diffusers.schedulers.scheduling_lms_discrete.LMSDiscreteScheduler),
# ("DDIMScheduler", diffusers.schedulers.scheduling_ddim.DDIMScheduler),
# ("DPMSolverMultistepScheduler", diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler),
# ("EulerDiscreteScheduler", diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler),
# ("PNDMScheduler", diffusers.schedulers.scheduling_pndm.PNDMScheduler),
# ("DDPMScheduler", diffusers.schedulers.scheduling_ddpm.DDPMScheduler),
# ("EulerAncestralDiscreteScheduler",
# diffusers.schedulers.scheduling_euler_ancestral_discrete.EulerAncestralDiscreteScheduler)
# ]
# loras = [
# (None, None),
# ("Night_scene_20230715120543", "../ai_files/loras/Night_scene_20230715120543.safetensors"),
# ("Oldsaigon-v1.0", "../ai_files/loras/Oldsaigon-v1.0.safetensors"),
# ("virginie_efira_v01", "../ai_files/loras/virginie_efira_v01.safetensors"),
# ]
guess_modes = [
# True, NotImplementedError: The operator 'aten::logspace.out' is not currently implemented for the MPS device
# lora_multipliers = [1.0, 2.0, 3, 0]
combined_list = list(itertools.product(
# schedulers,
# loras,
# lora_multipliers

for item in tqdm(combined_list, total=len(combined_list)):
base, (desc, image_path), (model_name, model_path), guess_mode = item
prompt: str = f"{base}, {desc}"
filename: str = f"{out_dir}/{base}_{desc}_{model_name}_{guess_mode}.png"
if not os.path.exists(filename):
controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny").to(device)
pipe = StableDiffusionControlNetPipeline.from_pretrained(
model_path, controlnet=controlnet,

# pipe.scheduler = scheduler.from_config(pipe.scheduler.config)
# if lora_name != None:
# device: str = "cpu"
# print(f"{lora_name} : {device}")
# pipe =
# pipe = load_lora_weights_orig(pipe, lora_file, lora_multiplier, device, torch.float32)
# else:
# device: str = "mps" if torch.backends.mps.is_available() else "cpu"
# print(f"No lora : {device}")
# pipe =
pipe =
factor: float = 0.5
init_image = get_canny_image(image_path)
width, height = init_image.size
new_width: int = math.floor(factor * width)
new_height: int = math.floor(factor * height)
init_image = init_image.resize((new_width, new_height))
images = pipe(prompt=prompt,
print(f"{filename} exist")

Here are some of the outputs.

anime_a sunset at the sea_flat2DAnimerge_False

flat2Danimerge is good, but usually add extra object to the picture.

anime_a sunset at the sea_kawaiiRealisticAnime_a03_False

`kawaiiRealisticAnime` looks realistic.

drawing_a sunset at the sea_exquisiteDetails_v10_False

`exquisiteDetails` I think it is too detail for called anime.

drawing_a sunset at the sea_flat2DAnimerge_False

`flat2DAnimerge` with prompt changing. It produces satisfied result.

water colour picture_a sunset at the sea_flat2DAnimerge_False

`flat2DAnimerge` with another prompt.




Sarit Ritwirune

On the way to full stack cross-platform. Currently make living by data science.