From 3acc016447279d1f0b748dfd1e62f402e7eedb49 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Sun, 12 Mar 2023 00:33:14 +0900 Subject: [PATCH 01/14] add mic input --- "app/01_\360\237\217\240_Home.py" | 51 +++++++++++++++++-- app/core.py | 4 +- .../02_\342\232\231\357\270\217_Settings.py" | 2 +- requirements.txt | 4 ++ 4 files changed, 54 insertions(+), 7 deletions(-) diff --git "a/app/01_\360\237\217\240_Home.py" "b/app/01_\360\237\217\240_Home.py" index a6ca088..3a47304 100644 --- "a/app/01_\360\237\217\240_Home.py" +++ "b/app/01_\360\237\217\240_Home.py" @@ -5,6 +5,9 @@ from config import get_page_config, get_whisper_settings, save_whisper_settings from core import MediaManager +from streamlit.runtime.uploaded_file_manager import UploadedFile, UploadedFileRec +from audiorecorder import audiorecorder + st.set_page_config(**get_page_config()) @@ -40,7 +43,7 @@ def get_formatted_date(date_str: str) -> str: # --------- with st.sidebar.expander("➕   Add Media", expanded=False): # # Render media type selection on the sidebar & the form - source_type = st.radio("Media Source", ["YouTube", "Upload"], label_visibility="collapsed") + source_type = st.radio("Media Source", ["YouTube", "Upload", "Mic"], label_visibility="collapsed") with st.form("input_form"): if source_type == "YouTube": youtube_url = st.text_input("Youtube video or playlist URL") @@ -48,6 +51,29 @@ def get_formatted_date(date_str: str) -> str: input_files = st.file_uploader( "Add one or more files", type=["mp4", "avi", "mov", "mkv", "mp3", "wav"], accept_multiple_files=True ) + elif source_type == "Mic": + audio = audiorecorder("Click to record", "Recording...") + if len(audio) > 0: + # To play audio in frontend: + #st.audio(audio.tobytes()) + + # To save audio to a file: + #wav_file = open("audio.mp3", "wb") + #wav_file.write(audio.tobytes()) + #wav_file.close() + + # emulate "uploaded file" + now = datetime.now() + d = f'{now:%Y%m%d%H%M%S}' + + filerec = UploadedFileRec(0, + f"mic_input_{d}.mp3", + "mp3", + audio.tobytes()) + input_files = UploadedFile(filerec) + else: + input_files = None + task_options = ["transcribe", "translate"] task = st.selectbox( "Task", options=task_options, index=task_options.index(st.session_state.whisper_params["task"]) @@ -66,6 +92,11 @@ def get_formatted_date(date_str: str) -> str: source = input_files else: st.error("Please upload files") + elif source_type == "Mic": + if input_files: + source = input_files + else: + st.error("Please input your voice via mic") # Lowercase the source type source_type = source_type.lower() @@ -156,7 +187,7 @@ def get_formatted_date(date_str: str) -> str: # Add a meta caption st.write(f"#### {media['source_name']}") - source_type = "YouTube" if media["source_type"] == "youtube" else "upload" + source_type = "YouTube" if media["source_type"] == "youtube" else media["source_type"] st.markdown( f""" Source: {source_type}
@@ -179,7 +210,7 @@ def get_formatted_date(date_str: str) -> str: # Render the media if media["source_type"] == "youtube": st.video(media["source_link"]) - elif media["source_type"] == "upload": + elif media["source_type"] in ["upload", "mic"]: st.audio(media["filepath"]) st.write("---") @@ -207,6 +238,12 @@ def get_formatted_date(date_str: str) -> str: # Add a meta caption source_type = "YouTube" if media["source_type"] == "youtube" else "uploaded" + if media["source_type"] == "youtube": + source_type = "YouTube" + elif media["source_type"] == "upload": + source_type = "uploaded" + else: + source_tpye = media["source_type"] st.markdown( f""" Source: {media['source_name']} ({source_type})
@@ -253,7 +290,7 @@ def get_formatted_date(date_str: str) -> str: if media["source_type"] == "youtube": st.sidebar.audio(media["filepath"], start_time=st.session_state.selected_media_offset) st.sidebar.video(media["source_link"]) - elif media["source_type"] == "upload": + elif media["source_type"] in ["upload", "mic"]: st.sidebar.audio(media["filepath"], start_time=st.session_state.selected_media_offset) st.write(f'## {media["source_name"]}') @@ -261,6 +298,12 @@ def get_formatted_date(date_str: str) -> str: with st.expander("📝   Metadata"): # Add a meta caption source_type = "YouTube" if media["source_type"] == "youtube" else "uploaded" + if media["source_type"] == "youtube": + source_type = "YouTube" + elif media["source_type"] == "upload": + source_type = "uploaded" + else: + source_tpye = media["source_type"] st.markdown( f""" Source: {media['source_name']} ({source_type})
diff --git a/app/core.py b/app/core.py index 6314fbd..d6db276 100644 --- a/app/core.py +++ b/app/core.py @@ -117,7 +117,7 @@ def _create(self, source_list: List[Union[str, Any]], source_type: str, whisper_ save_dir = self.media_dir / f"""{source_dirname}-{datetime.now().strftime("%Y-%m-%d %H-%M-%S")}""" # Download the audio file yc.streams.get_by_itag(140).download(save_dir, filename=save_filename) - elif source_type == "upload": + elif source_type in ["upload", "mic"]: # Parse the file name from the source tokens = source.name.split(".") source_name = ".".join(tokens[:-1]) @@ -162,7 +162,7 @@ def add(self, source: Union[str, Any], source_type: str, **whisper_args): source_list = list(Playlist(source)) else: source_list = [source] - elif source_type == "upload": + elif source_type in ["upload", "mic"]: source_list = source if type(source) == list else [source] # Download & save the audio files and add them to the database diff --git "a/app/pages/02_\342\232\231\357\270\217_Settings.py" "b/app/pages/02_\342\232\231\357\270\217_Settings.py" index 9e39b79..6d4c7f2 100644 --- "a/app/pages/02_\342\232\231\357\270\217_Settings.py" +++ "b/app/pages/02_\342\232\231\357\270\217_Settings.py" @@ -14,7 +14,7 @@ # -------------- st.write("### ⚙️ Whisper Settings") with st.form("whisper_settings_form"): - model_options = ["tiny", "base", "small", "medium", "large"] + model_options = ["tiny", "base", "small", "medium", "large", "large-v2"] selected_model = model_options.index(st.session_state.whisper_params["whisper_model"]) whisper_model = st.selectbox("Model", options=model_options, index=selected_model) temperature = st.number_input( diff --git a/requirements.txt b/requirements.txt index 9b16a64..c24f956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,7 @@ SQLAlchemy==2.0.3 # https://github.com/sqlalchemy/sqlalchemy # Frontend # -------------------- streamlit==1.18.1 # https://github.com/streamlit/streamlit + +# Audio Recorder on Frontend +# -------------------- +streamlit-audiorecorder==0.0.2 # https://github.com/theevann/streamlit-audiorecorder From 4b8ed0f3f36b4bf6ef0c38294212d110b2635179 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Mon, 13 Mar 2023 14:09:28 +0900 Subject: [PATCH 02/14] add fast-shutdown --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index cac6058..2f3946c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,5 +11,6 @@ services: # TODO: Fix this to relative paths from some user-defined directory - ./data:/data restart: unless-stopped + init: true # https://qiita.com/sun33/items/d62744f5746f6815f3c8 ports: - 8501:8501 From 7eb8cf5cdebea3891d99e1f78b3d72c8074bf392 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 14 Mar 2023 00:01:10 +0900 Subject: [PATCH 03/14] fix mic access issue --- .gitignore | 2 ++ .streamlit/config.toml | 7 +++++++ ssl/create-self-signed-cert.sh | 12 ++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 .streamlit/config.toml create mode 100644 ssl/create-self-signed-cert.sh diff --git a/.gitignore b/.gitignore index 0f8643d..515a067 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Custom ignore local/ data/ +ssl/server.crt +ssl/server.key ### Python template # Byte-compiled / optimized / DLL files diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000..db40277 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,7 @@ +[browser] +gatherUsageStats = false + +[server] +sslCertFile = './ssl/server.crt' +sslKeyFile = './ssl/server.key' + diff --git a/ssl/create-self-signed-cert.sh b/ssl/create-self-signed-cert.sh new file mode 100644 index 0000000..1f72f80 --- /dev/null +++ b/ssl/create-self-signed-cert.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +#openssl genrsa 2048 > server.key +#openssl req -new -key server.key > server.csr +#openssl x509 -days 3650 -req -sha256 -signkey server.key < server.csr > server.crt +#openssl x509 -text < server.crt + +# https://qiita.com/miyuki_samitani/items/b19aa5ac3b3c6e312bd5 +# https://qiita.com/sanyamarseille/items/46fc6ff5a0aca12e1946 + +# https://www.karakaram.com/creating-self-signed-certificate/ +openssl req -x509 -sha256 -nodes -days 3650 -newkey rsa:2048 -subj /CN=localhost -keyout server.key -out server.crt From 570ff8f59c9103f8fcac61c80733b13b5c0731c8 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 14 Mar 2023 08:24:28 +0900 Subject: [PATCH 04/14] update streamlit to support https --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c24f956..50882ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ SQLAlchemy==2.0.3 # https://github.com/sqlalchemy/sqlalchemy # Frontend # -------------------- -streamlit==1.18.1 # https://github.com/streamlit/streamlit +streamlit==1.20.0 # https://github.com/streamlit/streamlit # Audio Recorder on Frontend # -------------------- From 519b18a71bc3238cd6cbb6cfd1f3dab4358a3198 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 14 Mar 2023 08:42:36 +0900 Subject: [PATCH 05/14] update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6ac9b9c..c6e5393 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ conda env create -f environment.yml Note: If you're using a CPU-only machine, your runtime can be sped-up by using quantization implemented by [@MicellaneousStuff](https://github.com/MiscellaneousStuff) by swapping out `pip install openai-whisper` from `requirements.txt` and replacing it with their fork `pip install git+https://github.com/MiscellaneousStuff/whisper.git` (See related discussion here - https://github.com/hayabhay/whisper-ui/issues/20) +Note: If you want to use "mic input", due to browser media access issue, you should We should use SSL termination configuration in a reverse proxy or load balancer, or you can use streamlit [HTTPS support](https://docs.streamlit.io/library/advanced-features/https-support). streamlit 1.20.0 or later supports HTTPS. "mic input" data will be treated as "uploaded". + ## Usage Once you're set up, you can run the app with: From 86c06f66b405135733b057ab2fdf56fc9679018c Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 14 Mar 2023 09:02:28 +0900 Subject: [PATCH 06/14] update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c6e5393..e4b6c95 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ conda env create -f environment.yml Note: If you're using a CPU-only machine, your runtime can be sped-up by using quantization implemented by [@MicellaneousStuff](https://github.com/MiscellaneousStuff) by swapping out `pip install openai-whisper` from `requirements.txt` and replacing it with their fork `pip install git+https://github.com/MiscellaneousStuff/whisper.git` (See related discussion here - https://github.com/hayabhay/whisper-ui/issues/20) -Note: If you want to use "mic input", due to browser media access issue, you should We should use SSL termination configuration in a reverse proxy or load balancer, or you can use streamlit [HTTPS support](https://docs.streamlit.io/library/advanced-features/https-support). streamlit 1.20.0 or later supports HTTPS. "mic input" data will be treated as "uploaded". +Note: If you want to use "mic input", due to browser media access issue, you should use SSL termination configuration in a reverse proxy or load balancer, or you can use streamlit [HTTPS support](https://docs.streamlit.io/library/advanced-features/https-support). streamlit 1.20.0 or later supports HTTPS. "mic input" data will be treated as "uploaded". ## Usage From af0541563b896ce4a33a084d2eabf298f739bbec Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 14 Mar 2023 12:04:30 +0900 Subject: [PATCH 07/14] fix http-mode on docker --- .streamlit/config.toml | 7 ------- Dockerfile | 6 +++++- 2 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 .streamlit/config.toml diff --git a/.streamlit/config.toml b/.streamlit/config.toml deleted file mode 100644 index db40277..0000000 --- a/.streamlit/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[browser] -gatherUsageStats = false - -[server] -sslCertFile = './ssl/server.crt' -sslKeyFile = './ssl/server.key' - diff --git a/Dockerfile b/Dockerfile index 187535e..28e4504 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,8 @@ EXPOSE 8501 VOLUME /data # Run the app -CMD streamlit run /app/01_🏠_Home.py +#CMD streamlit run /app/01_🏠_Home.py + +# Run the app on https +COPY ssl /app/ssl +CMD streamlit run /app/01_🏠_Home.py --server.sslCertFile '/app/ssl/server.crt' --server.sslKeyFile '/app/ssl/server.key' From 1010ae2219833331ce36d2960d5f3fd1654f9391 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Wed, 15 Mar 2023 15:16:30 +0900 Subject: [PATCH 08/14] update environment.yml --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c3bf90b..5be667a 100644 --- a/environment.yml +++ b/environment.yml @@ -75,7 +75,8 @@ dependencies: - six==1.16.0 - smmap==5.0.0 - sqlalchemy==2.0.2 - - streamlit==1.17.0 + - streamlit==1.20.0 + - streamlit-audiorecorder==0.0.2 - tokenizers==0.13.2 - toml==0.10.2 - toolz==0.12.0 From bc58549f212654ce2a523c1a083f1cba5508ccb2 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Wed, 15 Mar 2023 15:21:08 +0900 Subject: [PATCH 09/14] chmod +x create-self-signed-cert.sh --- ssl/create-self-signed-cert.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ssl/create-self-signed-cert.sh diff --git a/ssl/create-self-signed-cert.sh b/ssl/create-self-signed-cert.sh old mode 100644 new mode 100755 From 15c37a4b7885f7cabe636248e00454e5758c06a4 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Thu, 6 Apr 2023 19:08:15 +0900 Subject: [PATCH 10/14] update README and fix proxy issue --- Dockerfile | 4 ++-- README.md | 14 +++++++++++--- "app/01_\360\237\217\240_Home.py" | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 28e4504..4b8fb17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,11 @@ COPY app /app COPY ./requirements.txt /requirements.txt # Pip install the dependencies -RUN pip install --upgrade pip +RUN pip --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org install --upgrade pip # For CPU only, you can use pip install git+https://github.com/MiscellaneousStuff/whisper.git # in place of openai-whisper. # Also, --extra-index-url https://download.pytorch.org/whl/cpu might be needed if you are using a CPU only machine -RUN pip install --no-cache-dir -r /requirements.txt +RUN pip install --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org --no-cache-dir -r /requirements.txt # Set the working directory to /app WORKDIR /app diff --git a/README.md b/README.md index e4b6c95..1ecd164 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,22 @@ conda env create -f environment.yml Note: If you're using a CPU-only machine, your runtime can be sped-up by using quantization implemented by [@MicellaneousStuff](https://github.com/MiscellaneousStuff) by swapping out `pip install openai-whisper` from `requirements.txt` and replacing it with their fork `pip install git+https://github.com/MiscellaneousStuff/whisper.git` (See related discussion here - https://github.com/hayabhay/whisper-ui/issues/20) -Note: If you want to use "mic input", due to browser media access issue, you should use SSL termination configuration in a reverse proxy or load balancer, or you can use streamlit [HTTPS support](https://docs.streamlit.io/library/advanced-features/https-support). streamlit 1.20.0 or later supports HTTPS. "mic input" data will be treated as "uploaded". +## Setup SSL termination configuration + +This `mic_support` branch supports "mic input". Due to browser media access issue, you should use SSL termination configuration in a reverse proxy or load balancer, or you can use streamlit [HTTPS support](https://docs.streamlit.io/library/advanced-features/https-support). streamlit 1.20.0 or later supports HTTPS. "mic input" data will be treated as "uploaded". + +As the easiest way, you can create a self-signed-cert as follows: + +``` +ssl/create-self-signed-cert.sh +``` ## Usage Once you're set up, you can run the app with: ``` -streamlit run app/01_🏠_Home.py +streamlit run app/01_🏠_Home.py --server.sslCertFile 'ssl/server.crt' --server.sslKeyFile 'ssl/server.key' ``` This will open a new tab in your browser with the app. You can then select a YouTube URL or local file & click "Run Whisper" to run the model on the selected media. @@ -45,7 +53,7 @@ Alternatively, you can run the app containerized with Docker via the included do docker compose up ``` -Then open up a new tab and navigate to [http://localhost:8501/](http://localhost:8501/) +Then open up a new tab and navigate to [https://localhost:8501/](https://localhost:8501/) NOTE: For existing users, this will break the database since absolute paths of files are saved. A future fix will be added to fix this. diff --git "a/app/01_\360\237\217\240_Home.py" "b/app/01_\360\237\217\240_Home.py" index 3a47304..32d0404 100644 --- "a/app/01_\360\237\217\240_Home.py" +++ "b/app/01_\360\237\217\240_Home.py" @@ -5,6 +5,9 @@ from config import get_page_config, get_whisper_settings, save_whisper_settings from core import MediaManager +import ssl +ssl._create_default_https.context = ssl._create_unverified_context + from streamlit.runtime.uploaded_file_manager import UploadedFile, UploadedFileRec from audiorecorder import audiorecorder From 944fbef142a3ab5cd4040b9a7627d7d3f12af2b2 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Thu, 6 Apr 2023 19:55:25 +0900 Subject: [PATCH 11/14] fix typo --- "app/01_\360\237\217\240_Home.py" | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git "a/app/01_\360\237\217\240_Home.py" "b/app/01_\360\237\217\240_Home.py" index 32d0404..6e39977 100644 --- "a/app/01_\360\237\217\240_Home.py" +++ "b/app/01_\360\237\217\240_Home.py" @@ -5,8 +5,9 @@ from config import get_page_config, get_whisper_settings, save_whisper_settings from core import MediaManager +# for self-signed-cert, not verified but un-safe. import ssl -ssl._create_default_https.context = ssl._create_unverified_context +ssl._create_default_https_context = ssl._create_unverified_context from streamlit.runtime.uploaded_file_manager import UploadedFile, UploadedFileRec from audiorecorder import audiorecorder From 3290d4dda059f178bac6605c971b82c029c770fa Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Fri, 7 Apr 2023 14:54:16 +0900 Subject: [PATCH 12/14] fix README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ecd164..c2f93a3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ This `mic_support` branch supports "mic input". Due to browser media access issu As the easiest way, you can create a self-signed-cert as follows: ``` -ssl/create-self-signed-cert.sh +cd ssl +create-self-signed-cert.sh ``` ## Usage From 0cf500bedaa020d2f40b7b5cf1e6fb5ddb68a541 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Tue, 11 Apr 2023 15:40:43 +0900 Subject: [PATCH 13/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c2f93a3..ba4164e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Streamlit UI for OpenAI's Whisper +# Streamlit UI for OpenAI's Whisper (mic support) This is a simple [Streamlit UI](https://streamlit.io/) for [OpenAI's Whisper speech-to-text model](https://openai.com/blog/whisper/). It let's you download and transcribe media from YouTube videos, playlists, or local files. From 31e27812278e85d0c943889afc06209ce54bdc61 Mon Sep 17 00:00:00 2001 From: Yasuhiro Morioka Date: Wed, 25 Oct 2023 10:41:25 +0900 Subject: [PATCH 14/14] add download buttons --- "app/01_\360\237\217\240_Home.py" | 7 +++++++ 1 file changed, 7 insertions(+) diff --git "a/app/01_\360\237\217\240_Home.py" "b/app/01_\360\237\217\240_Home.py" index 6e39977..9406587 100644 --- "a/app/01_\360\237\217\240_Home.py" +++ "b/app/01_\360\237\217\240_Home.py" @@ -299,6 +299,13 @@ def get_formatted_date(date_str: str) -> str: st.write(f'## {media["source_name"]}') + with st.expander("🔽   Downloads"): + fpath = str(Path(media["filepath"]).parent) + fname = str(Path(media["filepath"]).parent.name) + for ext in (".srt",".vtt",".json",".txt",".tsv"): + with open(fpath + '/transcript' + ext) as f: + st.download_button('Download ' + ext, f, file_name=fname+ext); + with st.expander("📝   Metadata"): # Add a meta caption source_type = "YouTube" if media["source_type"] == "youtube" else "uploaded"