code

!pip install -Uq diffusers ftfy accelerate

!pip install -Uq git+https://github.com/huggingface/transformers

import torch
from PIL import Image
import numpy as np
from diffusers import StableDiffusionDepth2ImgPipeline, StableDiffusionUpscalePipeline
import gc
import os
from io import BytesIO
from google.colab import files

pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(
   "stabilityai/stable-diffusion-2-depth",
   torch_dtype=torch.float16,
).to("cuda")

def dummy(images, **kwargs):
    return images, False
pipe.safety_checker = dummy


loaded = files.upload()
img_name = next(iter(loaded))
img = Image.open(BytesIO(loaded[img_name])).convert("RGB")
init_image  = img.resize((640,480))


loaded = files.upload()
dep_name = next(iter(loaded))
dep = Image.open(BytesIO(loaded[dep_name])).convert("RGB")
init_depth = dep.resize((640,480))

depthOut = init_depth.convert("L")
depthOut = np.expand_dims(depthOut, axis=0)


depthOut = np.expand_dims(depthOut, axis=0).repeat(4, axis=0)

depthOut = torch.from_numpy(depthOut)
depthOut = 2. * depthOut - 1.

depthOut.shape

# 3/ Perform model prediction 

#@title { run: "auto" }
prompt = "[GIVE_PROMPT}" 
strength=0.75 #@param {type:"slider", min:0, max:1, step:0.01}
num_inference_steps=87 #@param {type:"slider", min:1, max:100, step:1}
guidance_scale=7.5 #@param {type:"slider", min:0, max:10, step:0.5}

n_prompt = "bad, deformed, ugly, not accurate, blurry, negative, blurred"

image = pipe(
             prompt=prompt, 
             strength=strength,
             num_inference_steps=num_inference_steps,
             guidance_scale=guidance_scale,
             image=init_image, 
             depth_map=depthOut, 
             negative_prompt=n_prompt, 
             ).images[0]

import matplotlib.pyplot as plt

fig, axs = plt.subplots(1, 3, figsize=(25, 20))
axs[0].imshow(init_image)
axs[1].imshow(init_depth)
axs[2].imshow(image)