-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathprocess.py
146 lines (118 loc) · 6.29 KB
/
process.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import os
import cv2
import glob
import time
import numpy as np
import soundfile
from music21 import converter
import streamlit as st
import streamlit_ext as ste
from streamlit import session_state as state
from streamlit_player import st_player
from preprocess import preprocess
from inference import video_to_roll_inference, roll_to_midi_inference
from generate_score import generate_score
def process(key):
if key == 'url': video_path = "./data/inference/01.mp4"
else: video_path = "./data/inference/02.mp4"
video_info = dict()
cap = cv2.VideoCapture(video_path)
video_info['video_fps'] = int(cap.get(cv2.CAP_PROP_FPS))
video_info['video_duration'] = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) / video_info['video_fps'])
with st.form(f"{key}_select_range_form"):
video_info['video_select_range'] = st.slider(label="Select video range (second)", min_value=0, max_value=int(float(video_info['video_duration'])), step=10, value=(50, min(int(float(video_info['video_duration'])), 100)), key=f'{key}_silder')
video_info['video_select_frame'] = video_info['video_select_range'][1] * video_info['video_fps'] - video_info['video_select_range'][0] * video_info['video_fps']
col1, col2, col3, col4 = st.columns([2, 1, 1, 1])
with col1:
instrument = st.selectbox(
'Select Instrument',
('Acoustic Grand Piano', 'Bright Acoustic Piano', 'Electric Grand Piano', 'Glockenspiel'
'Accordion', 'Acoustic Guitar (nylon)', 'Fretless Bass', 'Celesta', 'Xylophone', 'Violin', 'Cello', 'Flute',
'Trombone', ))
with col2:
origin = st.checkbox(label="Original Video", key=f'{key}_origin_checkbox')
with col3:
maked = st.checkbox(label="Visualization", key=f'{key}_maked_checkbox')
with col4:
sheet = st.checkbox(label="Sheet Music", value=True, disabled=True, key=f'{key}_sheet_checkbox')
submit = st.form_submit_button(label="Submit")
if submit:
state.submit = True
with st.spinner("Data Preprocess in Progress..."):
frames_with5 = preprocess(video_info, key=key)
preprocess_success_msg = st.success("Data has been successfully preprocessed!")
with st.spinner("Roll Data Inference in Progress..."):
roll, logit, pm_wav, pm_roll = video_to_roll_inference(video_info, frames_with5, instrument)
np.save('./data/outputs/dump.npy', roll)
soundfile.write("./data/outputs/sound.wav", pm_wav, 16000, format='wav')
roll_inference_success_msg = st.success("Piano roll has been successfully inferenced!")
with st.spinner("Roll Data Postprocess in Progress..."):
midi, midi_wav, pm_midi = roll_to_midi_inference(video_info, logit, instrument)
midi_inference_success_msg = st.success("Piano roll has been successfully postprocessed!")
if maked:
with st.spinner("Generating video ..."):
os.system("python game.py")
video_file = open("./data/outputs/video.mp4", "rb")
video_bytes = video_file.read()
video_inference_success_msg = st.success("Video created successfully!")
state.video_bytes = video_bytes
else:
state.audio_bytes = midi_wav
time.sleep(0.2)
preprocess_success_msg.empty()
time.sleep(0.2)
roll_inference_success_msg.empty()
time.sleep(0.2)
midi_inference_success_msg.empty()
if maked:
time.sleep(0.2)
video_inference_success_msg.empty()
if state.submit:
if origin:
with st.expander(":film_projector: Origin Video"):
if key == 'url':
st_player(state.url_input, key="youtube_player")
else:
st.video(state.video_input, format="video/mp4")
if maked:
with st.expander(":film_frames: Output Video"):
st.video(state.video_bytes, format="video/mp4")
else:
with st.expander(":loud_sound: Output Audio"):
st.audio(state.audio_bytes, sample_rate=16000)
if sheet and submit:
state.sheet = True
with st.spinner("Generating Music Score..."):
output_dir = "./data/outputs"
os.makedirs(output_dir, exist_ok=True)
output_roll_midi_path = os.path.join(output_dir, "pm_roll.midi")
output_midi_path = os.path.join(output_dir, "pm.midi")
pm_roll.write(output_roll_midi_path)
pm_midi.write(output_midi_path)
score = generate_score(output_roll_midi_path)
roll_pdf = os.path.join(output_dir, "roll_sheet")
roll_png = os.path.join(output_dir, "roll_sheet")
converter.subConverters.ConverterLilypond().write(score, fmt='png', fp=roll_png, subformats='png')
converter.subConverters.ConverterLilypond().write(score, fmt='pdf', fp=roll_pdf, subformats='pdf')
state.sheet_file = True
for file in glob.glob(output_dir + "/*"):
if "png" not in file and "pdf" not in file:
os.remove(file)
if state.sheet and state.sheet_file:
with st.expander(":musical_note: Sheet Music"):
st.image("./data/outputs/roll_sheet.png")
col1, col2, _, _, _ = st.columns(5)
with col1:
state.download_click = ste.download_button(
label="pdf download",
data=open("./data/outputs/roll_sheet.pdf", 'rb').read(),
file_name="piano_sheet.pdf",
mime="application/pdf"
)
with col2:
state.download_click = ste.download_button(
label="png download",
data=open("./data/outputs/roll_sheet.png", 'rb').read(),
file_name="piano_sheet.png",
mime="image/png"
)