From 21f91c21e8ae93dc7b741250bc97092a15aaf409 Mon Sep 17 00:00:00 2001 From: CsBigDataHub Date: Sat, 30 May 2026 14:38:31 -0400 Subject: [PATCH 1/4] implement tls --- src/reqman/__init__.py | 20 +++++++- src/reqman/com.py | 7 +-- src/reqman/env.py | 112 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 126 insertions(+), 13 deletions(-) diff --git a/src/reqman/__init__.py b/src/reqman/__init__.py index 86a6324..a98b5b2 100644 --- a/src/reqman/__init__.py +++ b/src/reqman/__init__.py @@ -1019,8 +1019,25 @@ async def asyncReqExecute( # CREATE the REAL NEW SCOPE !!!!!! newenv=env.Scope( scope , EXPOSEDS) + base_path = gscope.path + if not base_path and self.parent.name and self.parent.name != "": + base_path = os.path.dirname(os.path.abspath(self.parent.name)) + + verify = False + try: + verify = env.create_ssl_verify(newenv, base_path) + except env.ResolveException as e: + ssl_error = str(e) + else: + ssl_error = None + # execute the request (newcore) - ex = await newenv.call(method,path,headers,body,newsaves,newtests, timeout=timeout, doc=doc or "", querys=querys, proxies=proxy, http=http) + if ssl_error is None: + ex = await newenv.call(method,path,headers,body,newsaves,newtests, timeout=timeout, doc=doc or "", querys=querys, proxies=proxy, http=http, verify=verify) + else: + r = com.ResponseError(f"Request can't be resolved: {ssl_error}") + ex = env.Exchange(method,path,headers=dict(headers), tests=newtests, saves=newsaves, doc=doc or "") + ex.treatment(newenv, r) ex.nolimit = self.nolimit #TODO: not beautiful !!! @@ -2169,4 +2186,3 @@ def __repr__(self): yml= "\n%s\n%s\n" % ( "#"*80,y) return re.sub(r"\{\{([\w_\-\d]+)\}\}", r"<<\1>>", yml) - diff --git a/src/reqman/com.py b/src/reqman/com.py index c6df94a..59e308a 100644 --- a/src/reqman/com.py +++ b/src/reqman/com.py @@ -24,6 +24,8 @@ logger = logging.getLogger(__name__) +VerifyTypes = T.Union[bool, str, ssl.SSLContext] + def jdumps(o, *a, **k): k["ensure_ascii"] = False # ~ k["default"]=serialize @@ -86,13 +88,13 @@ def __init__(self,url): ResponseError.__init__(self,f"Invalid {url}") -async def call(method, url:str, body: bytes=b'', headers:Headers=Headers(), timeout=None,proxies=None) -> Response: +async def call(method, url:str, body: bytes=b'', headers:T.Mapping[str, str]=Headers(), timeout=None,proxies=None, verify: VerifyTypes=False) -> Response: assert isinstance(body, bytes) try: async with httpx.AsyncClient( cookies=_COOKIES, - verify=False, + verify=verify, follow_redirects=False, proxy=proxies, ) as client: @@ -204,4 +206,3 @@ def call_simulation(http, method, url:str,body: bytes=b"", headers:dict={}): # print(e) # asyncio.run(t()) - diff --git a/src/reqman/env.py b/src/reqman/env.py index a8063c6..9cc43cc 100644 --- a/src/reqman/env.py +++ b/src/reqman/env.py @@ -23,6 +23,7 @@ import hashlib import logging import dotenv; dotenv.load_dotenv() +import ssl logger = logging.getLogger(__name__) @@ -112,7 +113,7 @@ def jpath(elem, path: str) -> T.Any: class Exchange: id:str="" #unique id to identify request structure (to be able to match in dual mode) - def __init__(self,method: str,url: str,body: bytes=b"",headers: com.Headers=com.Headers(), tests:list=[], saves:list=[], doc:str = ""): + def __init__(self,method: str,url: str,body: bytes=b"",headers: T.Mapping[str, str]=com.Headers(), tests:list=[], saves:list=[], doc:str = ""): assert isinstance(body, bytes) self.datetime = datetime.datetime.now() @@ -147,12 +148,12 @@ def __init__(self,method: str,url: str,body: bytes=b"",headers: com.Headers=com. def path(self): return urllib.parse.urlparse(self.url).path - async def call(self, timeout=None, proxies=None, http=None) -> com.Response: + async def call(self, timeout=None, proxies=None, http=None, verify: com.VerifyTypes=False) -> com.Response: t1 = datetime.datetime.now() if http is None: #real call on the web - r = await com.call(self.method,self.url,self.body,self.inHeaders,timeout=timeout, proxies=proxies) + r = await com.call(self.method,self.url,self.body,self.inHeaders,timeout=timeout, proxies=proxies, verify=verify) else: #simulate with http hook r = com.call_simulation(http,self.method,self.url,self.body,dict(self.inHeaders)) @@ -273,6 +274,104 @@ def isJson(s:str): pass +def _as_bool_or_path(value): + if isinstance(value, str): + v = value.strip() + if v.lower() in ["true", "yes", "on", "1"]: + return True + if v.lower() in ["false", "no", "off", "0", "none", "null", ""]: + return False + return v + return value + + +def _resolve_path(path, base_path=None): + if not isinstance(path, str) or not path: + return path + if os.path.isabs(path) or base_path is None: + return path + return os.path.abspath(os.path.join(base_path, path)) + + +def _create_unverified_client_context(): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def create_ssl_verify(scope, base_path=None): + """Build the httpx verify setting from reqman SSL config.""" + + sslconf = scope.get("ssl", {}) + if sslconf is None: + sslconf = {} + if not isinstance(sslconf, dict): + raise ResolveException("ssl should be a dict") + + def get_ssl_value(name, default=None): + if name in sslconf: + return sslconf[name] + return scope.get(name, default) + + verify = get_ssl_value("verify", None) + cacert = get_ssl_value("cacert", None) + cert = get_ssl_value("cert", None) + key = get_ssl_value("key", get_ssl_value("cert_key", None)) + password = get_ssl_value("password", get_ssl_value("cert_password", None)) + + if cacert is not None: + verify = cacert + if verify is None: + verify = False + + verify = _as_bool_or_path(scope.resolve_string_or_not(verify)) + key = scope.resolve_string_or_not(key) + password = scope.resolve_string_or_not(password) + + if isinstance(cert, dict): + key = scope.resolve_string_or_not(cert.get("key", key)) + password = scope.resolve_string_or_not(cert.get("password", password)) + cert = scope.resolve_string_or_not(cert.get("cert", cert.get("file", None))) + elif isinstance(cert, (list, tuple)): + parts = list(cert) + if len(parts) not in [1, 2, 3]: + raise ResolveException("cert should contain cert, optional key, optional password") + cert = scope.resolve_string_or_not(parts[0]) + if len(parts) >= 2: + key = scope.resolve_string_or_not(parts[1]) + if len(parts) == 3: + password = scope.resolve_string_or_not(parts[2]) + else: + cert = scope.resolve_string_or_not(cert) + + if not cert: + if isinstance(verify, str): + cafile = _resolve_path(verify, base_path) + try: + return ssl.create_default_context(cafile=cafile) + except Exception as e: + raise ResolveException(f"ssl verify error: {e}") + return bool(verify) + + cert = _resolve_path(cert, base_path) + key = _resolve_path(key, base_path) + if isinstance(verify, str): + cafile = _resolve_path(verify, base_path) + context_factory = lambda: ssl.create_default_context(cafile=cafile) + elif verify: + context_factory = ssl.create_default_context + else: + context_factory = _create_unverified_client_context + + try: + ctx = context_factory() + ctx.load_cert_chain(certfile=cert, keyfile=key or None, password=password or None) + return ctx + except Exception as e: + raise ResolveException(f"ssl cert error: {e}") + + class Scope(dict): # like 'Env' _locals_={} @@ -475,7 +574,7 @@ def run(self,method: T.Callable , value, with_env=True): raise PyMethodException(f"Can't execute '{method.__name__}' : {e}") - async def call(self, method:str, path:str, headers:dict={}, body:str="", saves=[], tests=[], doc:str="", timeout=None, querys={}, proxies=None, http=None) -> Exchange: + async def call(self, method:str, path:str, headers:dict={}, body:str="", saves=[], tests=[], doc:str="", timeout=None, querys={}, proxies=None, http=None, verify: com.VerifyTypes=False) -> Exchange: assert isinstance(body, str) # assert all( [isinstance(i, tuple) and len(i)==2 for i in tests] ) # assert list of tuple of 2 assert all( [isinstance(i, tuple) and len(i)==2 for i in saves] ) # assert list of tuple of 2 @@ -557,7 +656,7 @@ async def call(self, method:str, path:str, headers:dict={}, body:str="", saves=[ ex=Exchange(method,path,asBytes,dict(headers), tests=tests, saves=saves, doc=doc) ex.id=uid.hexdigest() #<- save unique id for this kind of request (to be able to match them in dual mode) if r is None: # we can call it safely - r=await ex.call(timeout,proxies=proxies,http=http) + r=await ex.call(timeout,proxies=proxies,http=http,verify=verify) ex.treatment( self, r) @@ -622,6 +721,3 @@ async def call(self, method:str, path:str, headers:dict={}, body:str="", saves=[ # pprint.pprint(ex.saves) # # print(r) - - - From 26363d8128284eb83611b0f4d669754bcac6637f Mon Sep 17 00:00:00 2001 From: CsBigDataHub Date: Sat, 30 May 2026 14:40:41 -0400 Subject: [PATCH 2/4] pls tests --- tests/test__newcore_com.py | 20 ++++++++ tests/test__newcore_env.py | 93 +++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/tests/test__newcore_com.py b/tests/test__newcore_com.py index fa10aac..ee2e342 100644 --- a/tests/test__newcore_com.py +++ b/tests/test__newcore_com.py @@ -110,6 +110,26 @@ async def test_call_json_decode_error_jules(mock_async_client): assert r.content == b'invalid json' +@pytest.mark.asyncio +@patch('httpx.AsyncClient') +async def test_call_passes_ssl_verify(mock_async_client): + mock_response = MagicMock() + mock_response.headers = {} + mock_response.content = b'ok' + mock_response.status_code = 200 + mock_response.reason_phrase = "OK" + mock_response.http_version = "HTTP/1.1" + mock_instance = mock_async_client.return_value.__aenter__.return_value + mock_instance.request.return_value = mock_response + + verify = object() + r = await reqman.com.call("GET", "https://test.com", verify=verify) + + assert r.status == 200 + mock_async_client.assert_called_once() + assert mock_async_client.call_args.kwargs["verify"] is verify + + @pytest.mark.asyncio @patch('httpx.AsyncClient') async def test_call_ssl_error_jules(mock_async_client): diff --git a/tests/test__newcore_env.py b/tests/test__newcore_env.py index 8921e19..033fb25 100644 --- a/tests/test__newcore_env.py +++ b/tests/test__newcore_env.py @@ -1,6 +1,7 @@ import pytest from src import reqman import json,os +from unittest.mock import MagicMock ENV=reqman.env.Scope(dict( txt="hello", @@ -230,6 +231,96 @@ def test_dotenv(): assert e.get_var("user") == "dotenv_ready" +def test_create_ssl_verify_defaults_to_current_unverified_behavior(): + e = reqman.env.Scope({}) + + assert reqman.env.create_ssl_verify(e) is False + + +def test_create_ssl_verify_uses_boolean_value(): + e = reqman.env.Scope(dict(verify=True)) + + assert reqman.env.create_ssl_verify(e) is True + + +def test_create_ssl_verify_uses_cacert(monkeypatch, tmp_path): + ca = tmp_path / "ca.pem" + ca.write_text("ca") + created = object() + + create_default_context = MagicMock(return_value=created) + monkeypatch.setattr(reqman.env.ssl, "create_default_context", create_default_context) + e = reqman.env.Scope(dict(cacert="ca.pem")) + + assert reqman.env.create_ssl_verify(e, str(tmp_path)) is created + create_default_context.assert_called_once_with(cafile=str(ca)) + + +def test_create_ssl_verify_loads_client_cert_and_key(monkeypatch, tmp_path): + cert = tmp_path / "client.pem" + key = tmp_path / "client.key" + cert.write_text("cert") + key.write_text("key") + ctx = MagicMock() + + monkeypatch.setattr(reqman.env, "_create_unverified_client_context", MagicMock(return_value=ctx)) + e = reqman.env.Scope(dict(cert=dict(cert="client.pem", key="client.key", password="secret"))) + + assert reqman.env.create_ssl_verify(e, str(tmp_path)) is ctx + ctx.load_cert_chain.assert_called_once_with( + certfile=str(cert), + keyfile=str(key), + password="secret", + ) + + +def test_create_ssl_verify_uses_ssl_grouping(monkeypatch, tmp_path): + ca = tmp_path / "ca.pem" + ca.write_text("ca") + created = object() + + create_default_context = MagicMock(return_value=created) + monkeypatch.setattr(reqman.env.ssl, "create_default_context", create_default_context) + e = reqman.env.Scope(dict(ssl=dict(cacert="ca.pem"))) + + assert reqman.env.create_ssl_verify(e, str(tmp_path)) is created + create_default_context.assert_called_once_with(cafile=str(ca)) + + +def test_create_ssl_verify_loads_client_cert_from_list(monkeypatch, tmp_path): + cert = tmp_path / "client.pem" + key = tmp_path / "client.key" + cert.write_text("cert") + key.write_text("key") + ctx = MagicMock() + + monkeypatch.setattr(reqman.env, "_create_unverified_client_context", MagicMock(return_value=ctx)) + e = reqman.env.Scope(dict(cert=["client.pem", "client.key", "secret"])) + + assert reqman.env.create_ssl_verify(e, str(tmp_path)) is ctx + ctx.load_cert_chain.assert_called_once_with( + certfile=str(cert), + keyfile=str(key), + password="secret", + ) + + +def test_create_ssl_verify_loads_client_cert_from_string(monkeypatch, tmp_path): + cert = tmp_path / "client.pem" + cert.write_text("cert") + ctx = MagicMock() + + monkeypatch.setattr(reqman.env, "_create_unverified_client_context", MagicMock(return_value=ctx)) + e = reqman.env.Scope(dict(cert="client.pem")) + + assert reqman.env.create_ssl_verify(e, str(tmp_path)) is ctx + ctx.load_cert_chain.assert_called_once_with( + certfile=str(cert), + keyfile=None, + password=None, + ) + + @pytest.mark.asyncio async def test_real_call(): # e=reqman.env.Env( {} ) @@ -279,4 +370,4 @@ def test_int_size(): assert e.resolve_all("42") == int("42") assert e.get_var("a.cc")==int("12") - assert e.get_var("a.cc.size")==2 \ No newline at end of file + assert e.get_var("a.cc.size")==2 From 7700f52e78fa08f2340c39da95aafa0e7834d739 Mon Sep 17 00:00:00 2001 From: CsBigDataHub Date: Sat, 30 May 2026 14:41:46 -0400 Subject: [PATCH 3/4] tls conf doc --- docs/conf.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/docs/conf.md b/docs/conf.md index e465036..969ce35 100644 --- a/docs/conf.md +++ b/docs/conf.md @@ -60,6 +60,95 @@ headers: content-type: application/json ``` +### "verify", "cacert" and "cert" +Reqman keeps its historical HTTPS behavior by default: SSL verification is disabled unless configured. + +Enable server certificate verification using the default trusted CA bundle: + +```yaml +verify: true +``` + +Use a custom CA bundle, equivalent to curl's `--cacert`: + +```yaml +cacert: ca.pem +``` + +Use a PEM client certificate, equivalent to curl's `--cert`: + +```yaml +cert: client.pem +``` + +Use a separate PEM client certificate and key, equivalent to curl's `--cert` and `--key`: + +```yaml +cert: + cert: client.pem + key: key.pem +``` + +Encrypted private keys can set `password`: + +```yaml +cert: + cert: client.pem + key: key.pem + password: secret +``` + +You can also group SSL options under `ssl`. This is preferred when setting a client key because `key` is otherwise a generic top-level name: + +```yaml +ssl: + verify: true + cacert: ca.pem + cert: client.pem + key: key.pem +``` + +Relative certificate paths are resolved from the folder containing `reqman.conf` or the scenario file. + +Full TLS example in a `reqman.conf`: + +```yaml +root: https://api.example.com +timeout: 10000 + +headers: + accept: application/json + content-type: application/json + +ssl: + # Optional when cacert is set; shown here to make verification explicit. + verify: true + + # Verify the server certificate against this private CA bundle. + cacert: certs/ca.pem + + # Send this client certificate/key pair for mutual TLS authentication. + cert: certs/client.pem + key: certs/client.key + + # Optional: only needed when the private key is encrypted. + password: secret +``` + +With this configuration, a scenario can use relative paths and inherit the TLS settings: + +```yaml +- GET: /health + tests: + - R.status: 200 + +- POST: /schemas + body: + name: customer + tests: + - R.status: [200, 201] +``` + ## Declare switches for command line ### "switches" Switches are a reqman's feature, to let you override default param with command line switches. @@ -118,5 +207,3 @@ It can be useful to obtain an oauth2 token bearer, initiate some things, ... It's a special procedure, which can be declared in reqman.conf only, and is auto-called at the end of all tests. It can be useful to clear some things after all tests, ... - - From 6be88e36670cd94e4a793ed48c33075be8bdd898 Mon Sep 17 00:00:00 2001 From: CsBigDataHub Date: Sat, 30 May 2026 14:46:28 -0400 Subject: [PATCH 4/4] tls real tests --- REALTESTS/auto_tls_config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 REALTESTS/auto_tls_config.yml diff --git a/REALTESTS/auto_tls_config.yml b/REALTESTS/auto_tls_config.yml new file mode 100644 index 0000000..d142e9d --- /dev/null +++ b/REALTESTS/auto_tls_config.yml @@ -0,0 +1,13 @@ +#:valid:1 THIS --b +# -*- coding: utf-8 -*- + +root: http://localhost:11111 + +ssl: + verify: true + +RUN: +- GET: /get_json + doc: TLS config can be declared globally and inherited by requests. + tests: + - R.status: 200