diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bc1188..154efd6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,3 @@ - exclude: | (?x) # NOT INSTALLABLE ADDONS diff --git a/base_ir_binary/README.rst b/base_ir_binary/README.rst new file mode 100644 index 0000000..bed4caa --- /dev/null +++ b/base_ir_binary/README.rst @@ -0,0 +1,54 @@ +============== +Base Ir Binary +============== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f2d6522aca9e1ecdcac676df7c86a617e2b0a150f34e6ac92af3855bb71b7326 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-Escodoo%2Fserver--addons-lightgray.png?logo=github + :target: https://github.com/Escodoo/server-addons/tree/14.0/base_ir_binary + :alt: Escodoo/server-addons + +|badge1| |badge2| |badge3| + + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Escodoo + +Maintainers +~~~~~~~~~~~ + +This module is part of the `Escodoo/server-addons `_ project on GitHub. + +You are welcome to contribute. diff --git a/base_ir_binary/__init__.py b/base_ir_binary/__init__.py new file mode 100644 index 0000000..1393216 --- /dev/null +++ b/base_ir_binary/__init__.py @@ -0,0 +1,3 @@ +from . import models + +from . import http diff --git a/base_ir_binary/__manifest__.py b/base_ir_binary/__manifest__.py new file mode 100644 index 0000000..59238b5 --- /dev/null +++ b/base_ir_binary/__manifest__.py @@ -0,0 +1,15 @@ +# Copyright 2023 - TODAY, Escodoo +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +{ + "name": "Base Ir Binary", + "summary": """ + Base ir.binary""", + "version": "14.0.1.0.0", + "license": "AGPL-3", + "author": "Escodoo", + "website": "https://github.com/Escodoo/server-addons", + "depends": [], + "data": [], + "demo": [], +} diff --git a/base_ir_binary/http.py b/base_ir_binary/http.py new file mode 100644 index 0000000..b39ed48 --- /dev/null +++ b/base_ir_binary/http.py @@ -0,0 +1,213 @@ +import base64 +import contextlib +import os +from io import BytesIO +from os.path import join as opj +from pathlib import Path +from zlib import adler32 + +import werkzeug.local +import werkzeug.utils + +from odoo.http import STATIC_CACHE_LONG, Response, request, root +from odoo.tools import config + +from .misc import file_path + +try: + from werkzeug.utils import send_file as _send_file +except ImportError: + from odoo.http import send_file as _send_file + + +class Stream: + """ + Send the content of a file, an attachment or a binary field via HTTP + + This utility is safe, cache-aware and uses the best available + streaming strategy. Works best with the --x-sendfile cli option. + + Create a Stream via one of the constructors: :meth:`~from_path`:, + :meth:`~from_attachment`: or :meth:`~from_binary_field`:, generate + the corresponding HTTP response object via :meth:`~get_response`:. + + Instantiating a Stream object manually without using one of the + dedicated constructors is discouraged. + """ + + type: str = "" # 'data' or 'path' or 'url' + data = None + path = None + url = None + + mimetype = None + as_attachment = False + download_name = None + conditional = True + etag = True + last_modified = None + max_age = None + immutable = False + size = None + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + @classmethod + def from_path(cls, path, filter_ext=("",)): + """Create a :class:`~Stream`: from an addon resource.""" + path = file_path(path, filter_ext) + check = adler32(path.encode()) + stat = os.stat(path) + return cls( + type="path", + path=path, + download_name=os.path.basename(path), + etag=f"{int(stat.st_mtime)}-{stat.st_size}-{check}", + last_modified=stat.st_mtime, + size=stat.st_size, + ) + + @classmethod + def from_attachment(cls, attachment): + """Create a :class:`~Stream`: from an ir.attachment record.""" + attachment.ensure_one() + + self = cls( + mimetype=attachment.mimetype, + download_name=attachment.name, + conditional=True, + etag=attachment.checksum, + ) + + if attachment.store_fname: + self.type = "path" + self.path = werkzeug.security.safe_join( + os.path.abspath(config.filestore(request.db)), attachment.store_fname + ) + stat = os.stat(self.path) + self.last_modified = stat.st_mtime + self.size = stat.st_size + + elif attachment.db_datas: + self.type = "data" + self.data = attachment.raw + self.last_modified = attachment["__last_update"] + self.size = len(self.data) + + elif attachment.url: + # When the URL targets a file located in an addon, assume it + # is a path to the resource. It saves an indirection and + # stream the file right away. + static_path = root.get_static_file( + attachment.url, host=request.httprequest.environ.get("HTTP_HOST", "") + ) + if static_path: + self = cls.from_path(static_path) + else: + self.type = "url" + self.url = attachment.url + + else: + self.type = "data" + self.data = b"" + self.size = 0 + + return self + + @classmethod + def from_binary_field(cls, record, field_name): + """Create a :class:`~Stream`: from a binary field.""" + data_b64 = record[field_name] + data = base64.b64decode(data_b64) if data_b64 else b"" + return cls( + type="data", + data=data, + etag=request.env["ir.attachment"]._compute_checksum(data), + last_modified=record["__last_update"] if record._log_access else None, + size=len(data), + ) + + # pylint: disable=method-required-super + def read(self): + """Get the stream content as bytes.""" + if self.type == "url": + raise ValueError("Cannot read an URL") + + if self.type == "data": + return self.data + + with open(self.path, "rb") as file: + return file.read() + + def get_response(self, as_attachment=None, immutable=None, **send_file_kwargs): + """ + Create the corresponding :class:`~Response` for the current stream. + + :param bool as_attachment: Indicate to the browser that it + should offer to save the file instead of displaying it. + :param bool immutable: Add the ``immutable`` directive to the + ``Cache-Control`` response header, allowing intermediary + proxies to aggressively cache the response. This option + also set the ``max-age`` directive to 1 year. + :param send_file_kwargs: Other keyword arguments to send to + :func:`odoo.tools._vendor.send_file.send_file` instead of + the stream sensitive values. Discouraged. + """ + assert self.type in ( + "url", + "data", + "path", + ), "Invalid type: {self.type!r}, should be 'url', 'data' or 'path'." + assert ( + getattr(self, self.type) is not None + ), "There is nothing to stream, missing {self.type!r} attribute." + + if self.type == "url": + return request.redirect(self.url, code=301, local=False) + + if as_attachment is None: + as_attachment = self.as_attachment + if immutable is None: + immutable = self.immutable + + send_file_kwargs = { + "mimetype": self.mimetype, + "as_attachment": as_attachment, + "download_name": self.download_name, + "conditional": self.conditional, + "etag": self.etag, + "last_modified": self.last_modified, + "max_age": STATIC_CACHE_LONG if immutable else self.max_age, + "environ": request.httprequest.environ, + "response_class": Response, + **send_file_kwargs, + } + + if self.type == "data": + return _send_file(BytesIO(self.data), **send_file_kwargs) + + # self.type == 'path' + send_file_kwargs["use_x_sendfile"] = False + if config["x_sendfile"]: + with contextlib.suppress(ValueError): # outside of the filestore + fspath = Path(self.path).relative_to( + opj(config["data_dir"], "filestore") + ) + x_accel_redirect = f"/web/filestore/{fspath}" + send_file_kwargs["use_x_sendfile"] = True + + res = _send_file(self.path, **send_file_kwargs) + + if immutable and res.cache_control: + res.cache_control["immutable"] = None # None sets the directive + + if "X-Sendfile" in res.headers: + res.headers["X-Accel-Redirect"] = x_accel_redirect + + # In case of X-Sendfile/X-Accel-Redirect, the body is empty, + # yet werkzeug gives the length of the file. This makes + # NGINX wait for content that'll never arrive. + res.headers["Content-Length"] = "0" + + return res diff --git a/base_ir_binary/misc.py b/base_ir_binary/misc.py new file mode 100644 index 0000000..d1525cd --- /dev/null +++ b/base_ir_binary/misc.py @@ -0,0 +1,104 @@ +import os +from contextlib import ContextDecorator + +import odoo +from odoo.tools import config + + +# pylint: disable=class-camelcase +class replace_exceptions(ContextDecorator): + """ + Hide some exceptions behind another error. Can be used as a function + decorator or as a context manager. + + .. code-block: + + @route('/super/secret/route', auth='public') + @replace_exceptions(AccessError, by=NotFound()) + def super_secret_route(self): + if not request.session.uid: + raise AccessError("Route hidden to non logged-in users") + ... + + def some_util(): + ... + with replace_exceptions(ValueError, by=UserError("Invalid argument")): + ... + ... + + :param exceptions: the exception classes to catch and replace. + :param by: the exception to raise instead. + """ + + def __init__(self, *exceptions, by): + if not exceptions: + raise ValueError("Missing exceptions") + + wrong_exc = next( + (exc for exc in exceptions if not issubclass(exc, Exception)), None + ) + if wrong_exc: + raise TypeError(f"{wrong_exc} is not an exception class.") + + self.exceptions = exceptions + self.by = by + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is not None and issubclass(exc_type, self.exceptions): + raise self.by from exc_value + + +def file_path(file_path, filter_ext=("",), env=None): + """Verify that a file exists under a known `addons_path` directory and return its full path. + + Examples:: + + >>> file_path('hr') + >>> file_path('hr/static/description/icon.png') + >>> file_path('hr/static/description/icon.png', filter_ext=('.png', '.jpg')) + + :param str file_path: absolute file path, or relative path within any `addons_path` + directory + :param list[str] filter_ext: optional list of supported extensions (lowercase, with + leading dot) + :param env: optional environment, required for a file path within a temporary + directory + created using `file_open_temporary_directory()` + :return: the absolute path to the file + :raise FileNotFoundError: if the file is not found under the known `addons_path` + directories + :raise ValueError: if the file doesn't have one of the supported extensions + (`filter_ext`) + """ + root_path = os.path.abspath(config["root_path"]) + addons_paths = odoo.addons.__path__ + [root_path] + if env and hasattr(env.transaction, "__file_open_tmp_paths"): + addons_paths += env.transaction.__file_open_tmp_paths + is_abs = os.path.isabs(file_path) + normalized_path = os.path.normpath(os.path.normcase(file_path)) + + if filter_ext and not normalized_path.lower().endswith(filter_ext): + raise ValueError("Unsupported file: " + file_path) + + # ignore leading 'addons/' if present, it's the final component of root_path, but + # may sometimes be included in relative paths + if normalized_path.startswith("addons" + os.sep): + normalized_path = normalized_path[7:] + + for addons_dir in addons_paths: + # final path sep required to avoid partial match + parent_path = os.path.normpath(os.path.normcase(addons_dir)) + os.sep + fpath = ( + normalized_path + if is_abs + else os.path.normpath( + os.path.normcase(os.path.join(parent_path, normalized_path)) + ) + ) + if fpath.startswith(parent_path) and os.path.exists(fpath): + return fpath + + raise FileNotFoundError("File not found: " + file_path) diff --git a/base_ir_binary/models/__init__.py b/base_ir_binary/models/__init__.py new file mode 100644 index 0000000..0ad7071 --- /dev/null +++ b/base_ir_binary/models/__init__.py @@ -0,0 +1 @@ +from . import ir_binary diff --git a/base_ir_binary/models/ir_binary.py b/base_ir_binary/models/ir_binary.py new file mode 100644 index 0000000..3878a06 --- /dev/null +++ b/base_ir_binary/models/ir_binary.py @@ -0,0 +1,297 @@ +import logging +from datetime import datetime +from mimetypes import guess_extension + +import werkzeug.http + +from odoo import models +from odoo.exceptions import MissingError, UserError +from odoo.http import request +from odoo.tools import file_open +from odoo.tools.image import image_guess_size_from_field_name, image_process +from odoo.tools.mimetypes import get_extension, guess_mimetype + +from ..http import Stream +from ..misc import replace_exceptions + +DEFAULT_PLACEHOLDER_PATH = "web/static/img/placeholder.png" +_logger = logging.getLogger(__name__) + + +class IrBinary(models.AbstractModel): + _name = "ir.binary" + _description = "File streaming helper model for controllers" + + def _find_record( + self, + xmlid=None, + res_model="ir.attachment", + res_id=None, + access_token=None, + ): + """ + Find and return a record either using an xmlid either a model+id + pair. This method is an helper for the ``/web/content`` and + ``/web/image`` controllers and should not be used in other + contextes. + + :param Optional[str] xmlid: xmlid of the record + :param Optional[str] res_model: model of the record, + ir.attachment by default. + :param Optional[id] res_id: id of the record + :param Optional[str] access_token: access token to use instead + of the access rights and access rules. + :returns: single record + :raises MissingError: when no record was found. + """ + record = None + if xmlid: + record = self.env.ref(xmlid, False) + elif res_id is not None and res_model in self.env: + record = self.env[res_model].browse(res_id).exists() + if not record: + raise MissingError( + f"No record found for xmlid={xmlid}, res_model={res_model}, id={res_id}" + ) + + record = self._find_record_check_access(record, access_token) + return record + + def _find_record_check_access(self, record, access_token): + if record._name == "ir.attachment": + return record.validate_access(access_token) + + record.check_access_rights("read") + record.check_access_rule("read") + return record + + def _record_to_stream(self, record, field_name): + """ + Low level method responsible for the actual conversion from a + model record to a stream. This method is an extensible hook for + other modules. It is not meant to be directly called from + outside or the ir.binary model. + + :param record: the record where to load the data from. + :param str field_name: the binary field where to load the data + from. + :rtype: odoo.http.Stream + """ + if record._name == "ir.attachment" and field_name in ( + "raw", + "datas", + "db_datas", + ): + return Stream.from_attachment(record) + + record.check_field_access_rights("read", [field_name]) + field_def = record._fields[field_name] + + # fields.Binary(attachment=False) or compute/related + if not field_def.attachment or field_def.compute or field_def.related: + return Stream.from_binary_field(record, field_name) + + # fields.Binary(attachment=True) + field_attachment = ( + self.env["ir.attachment"] + .sudo() + .search( + domain=[ + ("res_model", "=", record._name), + ("res_id", "=", record.id), + ("res_field", "=", field_name), + ], + limit=1, + ) + ) + if not field_attachment: + # pylint: disable=translation-required + raise MissingError("The related attachment does not exist.") + return Stream.from_attachment(field_attachment) + + def _get_stream_from( + self, + record, + field_name="raw", + filename=None, + filename_field="name", + mimetype=None, + default_mimetype="application/octet-stream", + ): + """ + Create a :class:odoo.http.Stream: from a record's binary field. + + :param record: the record where to load the data from. + :param str field_name: the binary field where to load the data + from. + :param Optional[str] filename: when the stream is downloaded by + a browser, what filename it should have on disk. By default + it is ``{model}-{id}-{field}.{extension}``, the extension is + determined thanks to mimetype. + :param Optional[str] filename_field: like ``filename`` but use + one of the record's char field as filename. + :param Optional[str] mimetype: the data mimetype to use instead + of the stored one (attachment) or the one determined by + magic. + :param str default_mimetype: the mimetype to use when the + mimetype couldn't be determined. By default it is + ``application/octet-stream``. + :rtype: odoo.http.Stream + """ + with replace_exceptions( + ValueError, by=UserError(f"Expected singleton: {record}") + ): + record.ensure_one() + + try: + field_def = record._fields[field_name] + except KeyError: + raise UserError(f"Record has no field {field_name!r}.") + if field_def.type != "binary": + raise UserError( + f"Field {field_def!r} is type {field_def.type!r} but " + f"it is only possible to stream Binary or Image fields." + ) + + stream = self._record_to_stream(record, field_name) + + if stream.type in ("data", "path"): + if mimetype: + stream.mimetype = mimetype + elif not stream.mimetype: + if stream.type == "data": + head = stream.data[:1024] + else: + with open(stream.path, "rb") as file: + head = file.read(1024) + stream.mimetype = guess_mimetype(head, default=default_mimetype) + + if filename: + stream.download_name = filename + elif filename_field in record: + stream.download_name = record[filename_field] + if not stream.download_name: + stream.download_name = f"{record._table}-{record.id}-{field_name}" + + stream.download_name = stream.download_name.replace("\n", "_").replace( + "\r", "_" + ) + if ( + not get_extension(stream.download_name) + and stream.mimetype != "application/octet-stream" + ): + stream.download_name += guess_extension(stream.mimetype) or "" + + return stream + + def _get_image_stream_from( + self, + record, + field_name="raw", + filename=None, + filename_field="name", + mimetype=None, + default_mimetype="image/png", + placeholder=None, + width=0, + height=0, + crop=False, + quality=0, + ): + """ + Create a :class:odoo.http.Stream: from a record's binary field, + equivalent of :meth:`~get_stream_from` but for images. + + In case the record does not exist or is not accessible, the + alternative ``placeholder`` path is used instead. If not set, + a path is determined via + :meth:`~odoo.models.BaseModel._get_placeholder_filename` which + ultimately fallbacks on ``web/static/img/placeholder.png``. + + In case the arguments ``width``, ``height``, ``crop`` or + ``quality`` are given, the image will be post-processed and the + ETags (the unique cache http header) will be updated + accordingly. See also :func:`odoo.tools.image.image_process`. + + :param record: the record where to load the data from. + :param str field_name: the binary field where to load the data + from. + :param Optional[str] filename: when the stream is downloaded by + a browser, what filename it should have on disk. By default + it is ``{table}-{id}-{field}.{extension}``, the extension is + determined thanks to mimetype. + :param Optional[str] filename_field: like ``filename`` but use + one of the record's char field as filename. + :param Optional[str] mimetype: the data mimetype to use instead + of the stored one (attachment) or the one determined by + magic. + :param str default_mimetype: the mimetype to use when the + mimetype couldn't be determined. By default it is + ``image/png``. + :param Optional[pathlike] placeholder: in case the image is not + found or unaccessible, the path of an image to use instead. + By default the record ``_get_placeholder_filename`` on the + requested field or ``web/static/img/placeholder.png``. + :param int width: if not zero, the width of the resized image. + :param int height: if not zero, the height of the resized image. + :param bool crop: if true, crop the image instead of rezising + it. + :param int quality: if not zero, the quality of the resized + image. + + """ + stream = None + try: + stream = self._get_stream_from( + record, field_name, filename, filename_field, mimetype, default_mimetype + ) + except UserError: + if request.params.get("download"): + raise + + if not stream or stream.size == 0: + if not placeholder: + placeholder = record._get_placeholder_filename(field_name) + stream = self._get_placeholder_stream(placeholder) + if (width, height) == (0, 0): + width, height = image_guess_size_from_field_name(field_name) + + if stream.type == "url": + return stream # Rezising an external URL is not supported + if isinstance(stream.etag, str): + stream.etag += f"-{width}x{height}-crop={crop}-quality={quality}" + + if isinstance(stream.last_modified, (int, float)): + stream.last_modified = datetime.utcfromtimestamp(stream.last_modified) + modified = werkzeug.http.is_resource_modified( + request.httprequest.environ, + etag=stream.etag if isinstance(stream.etag, str) else None, + last_modified=stream.last_modified, + ) + + if modified and (width or height or crop): + if stream.type == "path": + with open(stream.path, "rb") as file: + stream.type = "data" + stream.path = None + stream.data = file.read() + stream.data = image_process( + stream.data, + size=(width, height), + crop=crop, + quality=quality, + ) + stream.size = len(stream.data) + + return stream + + def _get_placeholder_stream(self, path=None): + if not path: + path = DEFAULT_PLACEHOLDER_PATH + return Stream.from_path(path, filter_ext=(".png", ".jpg")) + + def _placeholder(self, path=False): + if not path: + path = DEFAULT_PLACEHOLDER_PATH + with file_open(path, "rb", filter_ext=(".png", ".jpg")) as file: + return file.read() diff --git a/base_ir_binary/readme/CONTRIBUTORS.rst b/base_ir_binary/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000..e69de29 diff --git a/base_ir_binary/readme/DESCRIPTION.rst b/base_ir_binary/readme/DESCRIPTION.rst new file mode 100644 index 0000000..e69de29 diff --git a/base_ir_binary/readme/USAGE.rst b/base_ir_binary/readme/USAGE.rst new file mode 100644 index 0000000..e69de29 diff --git a/base_ir_binary/static/description/icon.png b/base_ir_binary/static/description/icon.png new file mode 100644 index 0000000..12ab005 Binary files /dev/null and b/base_ir_binary/static/description/icon.png differ diff --git a/base_ir_binary/static/description/index.html b/base_ir_binary/static/description/index.html new file mode 100644 index 0000000..816b200 --- /dev/null +++ b/base_ir_binary/static/description/index.html @@ -0,0 +1,408 @@ + + + + + + +Base Ir Binary + + + +
+

Base Ir Binary

+ + +

Beta License: AGPL-3 Escodoo/server-addons

+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Escodoo
    • +
+
+
+

Maintainers

+

This module is part of the Escodoo/server-addons project on GitHub.

+

You are welcome to contribute.

+
+
+
+ + diff --git a/setup/.setuptools-odoo-make-default-ignore b/setup/.setuptools-odoo-make-default-ignore new file mode 100644 index 0000000..207e615 --- /dev/null +++ b/setup/.setuptools-odoo-make-default-ignore @@ -0,0 +1,2 @@ +# addons listed in this file are ignored by +# setuptools-odoo-make-default (one addon per line) diff --git a/setup/README b/setup/README new file mode 100644 index 0000000..a63d633 --- /dev/null +++ b/setup/README @@ -0,0 +1,2 @@ +To learn more about this directory, please visit +https://pypi.python.org/pypi/setuptools-odoo diff --git a/setup/base_ir_binary/odoo/addons/base_ir_binary b/setup/base_ir_binary/odoo/addons/base_ir_binary new file mode 120000 index 0000000..84c71dd --- /dev/null +++ b/setup/base_ir_binary/odoo/addons/base_ir_binary @@ -0,0 +1 @@ +../../../../base_ir_binary \ No newline at end of file diff --git a/setup/base_ir_binary/setup.py b/setup/base_ir_binary/setup.py new file mode 100644 index 0000000..28c57bb --- /dev/null +++ b/setup/base_ir_binary/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)