is it possible to add .gif or .mp4 video insted of image/green/white background image?

[akashAD98]

I want to add .gif or .mp4 video in my background,instead of static image
@PeterL1n @

[akashAD98]

#199

[Asthestarsfalll]

You need to chage the bgr from constant to corresponding frame in gif or video.

    if (output_composition is not None) and (output_type == 'video'):
        bgr_source = VideoReader(bgr_dir, transform)
        bgr_reader = DataLoader(source, batch_size=bgr_source, pin_memory=True, num_workers=num_workers)
    try:
        with torch.no_grad():
            bar = tqdm(total=len(source), disable=not progress, dynamic_ncols=True)
            rec = [None] * 4
            for src, bgr in zip(reader, bgr_reader):

                if downsample_ratio is None:
                    downsample_ratio = auto_downsample_ratio(*src.shape[2:])

                src = src.to(device, dtype, non_blocking=True).unsqueeze(0) # [B, T, C, H, W]
                fgr, pha, *rec = model(src, *rec, downsample_ratio)

                if output_foreground is not None:
                    writer_fgr.write(fgr[0])
                if output_alpha is not None:
                    writer_pha.write(pha[0])
                if output_composition is not None:
                    if output_type == 'video':
                        com = fgr * pha + bgr * (1 - pha)
                    else:
                        fgr = fgr * pha.gt(0)
                        com = torch.cat([fgr, pha], dim=-3)
                    writer_com.write(com[0])
                
                bar.update(src.size(1))

[akashAD98]

@Asthestarsfalll im not able to get output according to this, I added this code into inference.py
& passed video as input for background in bgr_dir & getting same green background instead of my custom video.

    if (output_composition is not None) and (output_type == 'video'):
        bgr_dir="/content/istock10minvide.mp4"
        bgr_source = VideoReader(bgr_dir, transform)
        bgr_reader = DataLoader(source, batch_size=bgr_source, pin_memory=True, num_workers=num_workers)

I used this command

from inference import convert_video

convert_video(
    model,                           # The model, can be on any device (cpu or cuda).
    input_source='/content/opvid_10_22.mp4',        # A video file or an image sequence directory.
    output_type='video',             # Choose "video" or "png_sequence"
    output_composition='/content/drive/MyDrive/Takeout/com.mp4',    # File path if video; directory path if png sequence.
    #output_alpha="pha.mp4",          # [Optional] Output the raw alpha prediction.
    #output_foreground="fgr.mp4",     # [Optional] Output the raw foreground prediction.
    output_video_mbps=4,             # Output video mbps. Not needed for png sequence.
    downsample_ratio=None,           # A hyperparameter to adjust or use None for auto.
    seq_chunk=12,                    # Process n frames at once for better parallelism.
)

[akashAD98]

PeterL1n/BackgroundMattingV2#191
@Asthestarsfalll can you please help me ?

[Asthestarsfalll]

@akashAD98
Can you upload the codes?
It seems that you did not use your own video, cause the way of composing video is unchanged.
Make sure you modify like this:

[akashAD98]

@Asthestarsfalll I have assigned path of my video ,still not working
line 116 I have given the path of the video bgr_dir ="op_bg.mp4"

inference_rvm_video.text.zip

[Asthestarsfalll]

@akashAD98
oh, just modify like this
bgr_reader = DataLoader(bgr_source, batch_size=seq_chunk, pin_memory=True, num_workers=num_workers)

[akashAD98]

@Asthestarsfalll i did changes still getting same op with green bg

[Asthestarsfalll]

@akashAD98
My code:

    if (output_composition is not None) and (output_type == 'video'):
        # bgr = torch.tensor([120, 255, 155], device=device, dtype=dtype).div(255).view(1, 1, 3, 1, 1)
        bgr_source = VideoReader("./asianboss.mp4", transform)
        bgr_reader = DataLoader(bgr_source, batch_size=seq_chunk, pin_memory=True, num_workers=num_workers)
    try:
        with torch.no_grad():
            bar = tqdm(total=len(source), disable=not progress, dynamic_ncols=True)
            rec = [None] * 4
            for (src, bgr) in zip(reader, bgr_reader):

                if downsample_ratio is None:
                    downsample_ratio = auto_downsample_ratio(*src.shape[2:])

                src = src.to(device, dtype, non_blocking=True).unsqueeze(0) # [B, T, C, H, W]
                fgr, pha, *rec = model(src, *rec, downsample_ratio)

                if output_foreground is not None:
                    writer_fgr.write(fgr[0])
                if output_alpha is not None:
                    writer_pha.write(pha[0])
                if output_composition is not None:
                    if output_type == 'video':
                        com = fgr * pha + bgr * (1 - pha)
                    else:
                        fgr = fgr * pha.gt(0)
                        com = torch.cat([fgr, pha], dim=-3)
                    writer_com.write(com[0])
                bar.update(src.size(1))

result:

[akashAD98]

@Asthestarsfalll are you using this repo https://github.com/PeterL1n/RobustVideoMatting ??
& this command? i did all changes inside the inference.py ,I'm correct ?

from inference import convert_video

convert_video(
    model,                           # The model, can be on any device (cpu or cuda).
    input_source='input.mp4',        # A video file or an image sequence directory.
    output_type='video',             # Choose "video" or "png_sequence"
    output_composition='com.mp4',    # File path if video; directory path if png sequence.
    output_alpha="pha.mp4",          # [Optional] Output the raw alpha prediction.
    output_foreground="fgr.mp4",     # [Optional] Output the raw foreground prediction.
    output_video_mbps=4,             # Output video mbps. Not needed for png sequence.
    downsample_ratio=None,           # A hyperparameter to adjust or use None for auto.
    seq_chunk=12,                    # Process n frames at once for better parallelism.
)

[akashAD98]

very strenge @Asthestarsfalll
i tried it with your code ..still had the same issue. this is my google collab, please can you check
RVM_withcustom_video_bg.zip

[akashAD98]

i solve issue. the command i need to run is this

!python inference.py \
    --variant resnet50 \
    --checkpoint "/content/RobustVideoMatting/CHECKPOINT/rvm_resnet50.pth" \
    --device cuda \
    --input-source "/content/vidforbg.mp4" \
    --output-type video \
    --output-composition "composition_opop_ISTOCK.mp4" \
    #--output-alpha "alpha.mp4" \
    #--output-foreground "foreground.mp4" \
    --output-video-mbps 4 \
    --seq-chunk 12

also make sure if you're running it on google collab. you will get 1 error

Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!

then just add cuda to bgr

bgr=bgr.to(device)

@Asthestarsfalll thanks for your help mate

[akashAD98]

it has one issue we need to resize both videos & also there duration

[Asthestarsfalll]

@akashAD98

1. Use the same transform:

        bgr_source = VideoReader("xxx.mp4", transform)  # there, the variable 'transform' is defined before

2. Use SequentialSampler may be helpful. Or just use a counter to control when to end the inference loop

[akashAD98]

@Asthestarsfalll yes but im already using transform here

RobustVideoMatting/inference_utils.py

Line 55 in 81a1093

class ImageSequenceReader(Dataset):

, I& SequentialSampler are same right?

[akashAD98]

@Asthestarsfalll

duration_bgr = len(bgr_reader)
print("duration/length of backgrVID",duration_bgr)

with torch.no_grad():
            bar = tqdm(total=len(source), disable=not progress, dynamic_ncols=True)

            ##ImageSequenceReader

            counter=0
            rec = [None] * 4
            for (src, bgr) in zip(reader, bgr_reader):

                if downsample_ratio is None:
                    downsample_ratio = auto_downsample_ratio(*src.shape[2:])
                counter+=1
                print(counter)
                ### datset
                if counter >= duration_bgr:
                     break

im getting this error

#######
duration/length of backgrVID 1753
size
  0% 0/353 [00:00<?, ?it/s]1
Traceback (most recent call last):
  File "inference.py", line 228, in <module>
    converter.convert(
  File "inference.py", line 204, in convert
    convert_video(self.model, device=self.device, dtype=torch.float32, *args, **kwargs)
  File "inference.py", line 170, in convert_video
    com = fgr * pha + bgr * (1 - pha)
RuntimeError: The size of tensor a (720) must match the size of tensor b (1920) at non-singleton dimension 4
  0% 0/353 [00:01<?, ?it/s]

[Asthestarsfalll]

@akashAD98
It seem that you did not specify the augment input-resize. So the transform will be ToTensor without resize by default.
Add resize to transform by your own or specify input-resize

    if input_resize is not None:
        transform = transforms.Compose([
            transforms.Resize(input_resize[::-1]),
            transforms.ToTensor()
        ])
    else:
        transform = transforms.ToTensor()

[akashAD98]

@Asthestarsfalll solve problem thanks mate

[darwinOne]

@akashAD98 Sir, can I see your code running successfully on colab?

[akashAD98]

@darwinOne yes it’s working fine

[darwinOne]

@akashAD98 can you share the code file, please?

[akashAD98]

@darwinOne yes please drop your mail

[FONCHIEH]

@darwinOne yes please drop your mail是的，请投递您的邮件

[email protected]
thanks