From bb61dac2b32f32d4cb3b6c6ecc470597c1903647 Mon Sep 17 00:00:00 2001 From: geniusLAT Date: Wed, 15 May 2024 20:06:47 +0500 Subject: [PATCH 01/12] dvc init happened --- .dvc/.gitignore | 3 +++ .dvc/config | 0 .dvcignore | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 .dvc/.gitignore create mode 100644 .dvc/config create mode 100644 .dvcignore diff --git a/.dvc/.gitignore b/.dvc/.gitignore new file mode 100644 index 0000000..528f30c --- /dev/null +++ b/.dvc/.gitignore @@ -0,0 +1,3 @@ +/config.local +/tmp +/cache diff --git a/.dvc/config b/.dvc/config new file mode 100644 index 0000000..e69de29 diff --git a/.dvcignore b/.dvcignore new file mode 100644 index 0000000..5197305 --- /dev/null +++ b/.dvcignore @@ -0,0 +1,3 @@ +# Add patterns of files dvc should ignore, which could improve +# the performance. Learn more at +# https://dvc.org/doc/user-guide/dvcignore From 8b3701bd62bf07da0e689a9eba91ebd71e97644e Mon Sep 17 00:00:00 2001 From: geniusLAT Date: Mon, 20 May 2024 16:47:30 +0500 Subject: [PATCH 02/12] test_data.csv.dvc added to dvc --- .dvc/config | 6 ++++++ .gitignore | 1 + test_data.csv.dvc | 5 +++++ 3 files changed, 12 insertions(+) create mode 100644 .gitignore create mode 100644 test_data.csv.dvc diff --git a/.dvc/config b/.dvc/config index e69de29..b601ef6 100644 --- a/.dvc/config +++ b/.dvc/config @@ -0,0 +1,6 @@ +[core] + remote = myminio +['remote "myminio"'] + url = s3://185.84.163.5:9000/dvcbucket2 + access_key_id= yZ9qhDgPHuqaRpQZ4kgH + secret_access_key= IksmsR3BHMCJQO9R9hqH72n7kMRHTG7xAlsRlpZN \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7acf8d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/test_data.csv diff --git a/test_data.csv.dvc b/test_data.csv.dvc new file mode 100644 index 0000000..223a459 --- /dev/null +++ b/test_data.csv.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 7d9ee4c06927ceccef9dc27350e015c9 + size: 52 + hash: md5 + path: test_data.csv From 3893099521bbab766630ad75f6d9202013fe90f8 Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:32:28 +0300 Subject: [PATCH 03/12] Delete .github/workflows/python-app2.yml --- .github/workflows/python-app2.yml | 39 ------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/python-app2.yml diff --git a/.github/workflows/python-app2.yml b/.github/workflows/python-app2.yml deleted file mode 100644 index b37004a..0000000 --- a/.github/workflows/python-app2.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python application - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f src/requirements.txt ]; then pip install -r src/requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest src/test.py From fe2285b295b67c5a31740f7ddb5eb6f0b6fea0ff Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:38:35 +0300 Subject: [PATCH 04/12] Update web_app.py --- active/web_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/active/web_app.py b/active/web_app.py index 55b6308..ee9a146 100644 --- a/active/web_app.py +++ b/active/web_app.py @@ -12,6 +12,7 @@ def process_text(input_text): return str(model.analyse(input_text)[0]) + # Обрабатываем введенный текст и выводим результат if st.button('Показать результат'): result = process_text(input_text) From ab22c347ce9cf58badd95ba5e6f2286600113843 Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:43:26 +0300 Subject: [PATCH 05/12] Create python-app.yml --- .github/workflows/python-app.yml | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..1168bd9 --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python application + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From 08ad099f838c71c4bfe05ec86dfd9bd22db6ffde Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:45:02 +0300 Subject: [PATCH 06/12] Update python-app.yml --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 1168bd9..164e2ca 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,9 +5,9 @@ name: Python application on: push: - branches: [ "main" ] + branches: [ "dvc_deploy" ] pull_request: - branches: [ "main" ] + branches: [ dvc_deploy" ] permissions: contents: read From 0ea1a2a8b89074287527714b430f93a76a138b93 Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:46:57 +0300 Subject: [PATCH 07/12] Update model.py --- active/model.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/active/model.py b/active/model.py index 33b8b38..0210539 100644 --- a/active/model.py +++ b/active/model.py @@ -1,8 +1,8 @@ from transformers import pipeline clf = pipeline( - task = 'sentiment-analysis', - model = 'SkolkovoInstitute/russian_toxicity_classifier') + task='sentiment-analysis', + model='SkolkovoInstitute/russian_toxicity_classifier') text = ['У нас в есть убунты и текникал превью.', 'Как минимум два малолетних дегенерата в треде, мда.', @@ -10,9 +10,10 @@ def analyse(text): - text=[text] - result=clf(text) + text = [text] + result = clf(text) print(result) return result -print(analyse("Кто любит жаб? Они крутые.")) \ No newline at end of file + +print(analyse("Кто любит жаб? Они крутые.")) From 4cbf3de5cd3055c27ba72da4645993c7558b7e1c Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:48:54 +0300 Subject: [PATCH 08/12] Update fa.py --- active/fa.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/active/fa.py b/active/fa.py index bb8fc6d..c0d51ca 100644 --- a/active/fa.py +++ b/active/fa.py @@ -1,12 +1,16 @@ from fastapi import FastAPI +import model app = FastAPI() -import model + def process(input_text): return str(model.analyse(input_text)[0]) + @app.get("/process_text/{text}") + + async def process_text(text: str): result = process(text) return {"result": result} From 412ce13858cb34aca4b96bf2347223d09474944d Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:49:54 +0300 Subject: [PATCH 09/12] Update fa.py --- active/fa.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/active/fa.py b/active/fa.py index c0d51ca..3c13c10 100644 --- a/active/fa.py +++ b/active/fa.py @@ -9,8 +9,6 @@ def process(input_text): @app.get("/process_text/{text}") - - async def process_text(text: str): result = process(text) return {"result": result} From 19ca7b8a6de03b22c0a8d1130e2e2e24c7ae9569 Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:50:48 +0300 Subject: [PATCH 10/12] Update model.py --- active/model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/active/model.py b/active/model.py index 0210539..269a30a 100644 --- a/active/model.py +++ b/active/model.py @@ -1,12 +1,10 @@ from transformers import pipeline clf = pipeline( - task='sentiment-analysis', + task='sentiment-analysis', model='SkolkovoInstitute/russian_toxicity_classifier') -text = ['У нас в есть убунты и текникал превью.', - 'Как минимум два малолетних дегенерата в треде, мда.', - 'иди на хер'] +text = ['У нас в есть убунты и текникал превью.', 'Как минимум два малолетних дегенерата в треде, мда.', 'иди на хер'] def analyse(text): From 61929441548ef4a599ce393e19ecc5bee92413f2 Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:53:29 +0300 Subject: [PATCH 11/12] Update main.py --- active/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/active/main.py b/active/main.py index f4cc0d5..f0d6f5e 100644 --- a/active/main.py +++ b/active/main.py @@ -1,4 +1,3 @@ - import os import uvicorn @@ -6,4 +5,4 @@ try: uvicorn.run("fa:app", host="0.0.0.0", port=int(os.environ['PORT']), reload=False, log_level="debug") except Exception as e: - print("Error on running uvicorn: "+str(e)) \ No newline at end of file + print("Error on running uvicorn: "+str(e)) From 6d5ad006e22471d4a3ddf63efc9f717bf27cc44d Mon Sep 17 00:00:00 2001 From: Moshwen <149946493+Moshwen@users.noreply.github.com> Date: Sat, 22 Jun 2024 18:00:49 +0300 Subject: [PATCH 12/12] Update main.py --- src/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.py b/src/main.py index cac7800..632227b 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,3 @@ - -import os import uvicorn if __name__ == "__main__":