diff --git a/.dockerignore b/.dockerignore index 842ed4a..946d3bb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,8 @@ README.md license.txt Dockerfile Makefile -*_matomo \ No newline at end of file +*_matomo +bin/ +env/ +lib/ +pyvenv.cfg \ No newline at end of file diff --git a/.env.example b/.env.example index d4ec0c6..02860d3 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ -base_url=enter_your_url_here -db_name=file_name_here -id_site=id_of_your_site -start_date=YYYY-MM-DD -token_auth=your_matomo_api_token \ No newline at end of file +JWT_SECRET_KEY=JWT_SECRET_KEY +POSTGRES_USER=POSTGRES_USER +POSTGRES_PASSWORD=POSTGRES_PASSWORD +POSTGRES_HOST=POSTGRES_HOST +POSTGRES_PORT=POSTGRES_PORT \ No newline at end of file diff --git a/.gitignore b/.gitignore index fdca2af..93ee583 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,8 @@ path **.pyc .env .secrets -coverage.xml \ No newline at end of file +coverage.xml +bin/ +env/ +lib/ +pyvenv.cfg \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ba69c56..8c4080c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,22 @@ -FROM ubuntu:latest -ENV PYTHONUNBUFFERED 1 -ARG DEBIAN_FRONTEND=noninteractive +FROM python:3.11.7-alpine3.19 + +ENV PORT=8080 \ + TIMEOUT=1200 + WORKDIR /workdir -RUN apt-get update -RUN apt-get -y upgrade -RUN apt-get install -y python3 -RUN apt-get install -y python3-pip -RUN apt-get install -y poppler-utils -RUN apt-get install -y libpq-dev -RUN apt-get install -y libsm6 libxext6 libxrender-dev -RUN apt-get install -y postgresql + +RUN apk update && apk add build-base \ + poppler-utils \ + libpq-dev \ + libsm-dev libxrender libxext-dev \ + postgresql \ + vim COPY requirements.txt . + RUN pip3 install -r requirements.txt + COPY . . -EXPOSE 8080 -ENV PORT 8080 -ENV TIMEOUT 1200 + +EXPOSE ${PORT} CMD gunicorn -w 4 --bind=0.0.0.0:${PORT} app:app --timeout=${TIMEOUT} diff --git a/app.py b/app.py index a82445c..37270d1 100644 --- a/app.py +++ b/app.py @@ -51,8 +51,12 @@ def wrapped(*args, **kwargs): def index(): data = request.args try: + print("request incoming") + print(data) main.exec(data) - except Exception: + except Exception as e: + print("this is the error") + print(e) return jsonify( {'message': 'Error executing script: recheck database variables'} ), 403 diff --git a/main.py b/main.py index bb6bd8d..5bc6f63 100644 --- a/main.py +++ b/main.py @@ -11,10 +11,14 @@ def exec(raw_database_variables=None): if os.path.exists('.env'): raw_database_variables = dotenv_values() + print("settings") + print(raw_database_variables) settings.init('config.yml', raw_database_variables) + print(settings.config['requests']) data_objects = data_handling.set_data_objects_for_sql_conversion( settings.config['requests'] ) + print(data_objects) sql_handling.fill_database(data_objects) diff --git a/matomo_pull/data_handling.py b/matomo_pull/data_handling.py index 12e9671..172dbd9 100644 --- a/matomo_pull/data_handling.py +++ b/matomo_pull/data_handling.py @@ -4,6 +4,8 @@ def set_data_objects_for_sql_conversion(reports_map): data_objects = {} + + print("we are here") for table_name, table_parameters in reports_map.items(): data_objects[table_name] = set_data_object_from_url( table_name, diff --git a/matomo_pull/settings.py b/matomo_pull/settings.py index 0d58e47..aa05727 100644 --- a/matomo_pull/settings.py +++ b/matomo_pull/settings.py @@ -8,6 +8,7 @@ def init(data_file='config.yml', raw_database_variables={}): global http, config, remote_database_variables, connection http = set_http_manager() config = set_config(data_file) + remote_database_variables = set_remote_database_variables( raw_database_variables ) @@ -33,6 +34,7 @@ def set_config(config_file='config.yml'): def set_remote_database_variables(data={}): + print(data) remote_database_variables = { 'base_url': data['base_url'], 'db_name': data['db_name'], @@ -69,16 +71,19 @@ def set_remote_database_variables(data={}): def set_database_connection(vars=None): try: + print(f"postgresql://{vars['POSTGRES_USER']}:{vars['POSTGRES_PASSWORD']}") connection = sqlalchemy.create_engine( f"postgresql://{vars['POSTGRES_USER']}:{vars['POSTGRES_PASSWORD']}" f"@{vars['POSTGRES_HOST']}:{vars['POSTGRES_PORT']}" f"/{vars['db_name']}" ) + print(connection) connection.connect() - except Exception: + except Exception as e: + print(e) raise ValueError( f"The Postgres database was wrongly configured." f"Available variables are {vars}." ) - return connection + return connection \ No newline at end of file diff --git a/matomo_pull/sql_handling.py b/matomo_pull/sql_handling.py index bc3761e..fc19109 100644 --- a/matomo_pull/sql_handling.py +++ b/matomo_pull/sql_handling.py @@ -2,7 +2,7 @@ from . import settings as s -def fill_database(data_objects): +def fill_database(data_objects): for table_name, data_object in data_objects.items(): convert_data_object_to_sql( table_name, diff --git a/matomo_pull/url_handling.py b/matomo_pull/url_handling.py index 3054f8f..95ea3a6 100644 --- a/matomo_pull/url_handling.py +++ b/matomo_pull/url_handling.py @@ -35,8 +35,10 @@ def set_basic_url_with_env_variables(): def http_get(url): try: + print(s.http.request("GET", url).headers) response = json.loads(s.http.request("GET", url).data.decode('utf-8')) - except Exception: + except Exception as e: + print(e) raise ValueError(f"Request returns unhandable data: {url}") if isinstance(response, dict) and response.get('result'): diff --git a/requirements.txt b/requirements.txt index 9dadb99..8b25207 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,8 @@ pytest-cov~=2.11.1 pytest-env~=0.6.2 pytest~=6.2.2 python-dotenv~=0.18.0 -pyyaml~=5.4.1 +pyyaml~=5.3.1 SQLAlchemy~=1.3.23 urllib3~=1.26.3 flask~=1.1.2 +markupsafe==2.0.1 \ No newline at end of file