diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..1f45749 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,27 @@ +### Type of Change + +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] Task + +### Description + +### TESTS + +Number of tests added/changed: + +### ADDITIONAL INFORMATION + + +- [ ] Removes existing feature +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] This change requires a documentation update + +### Checklist: +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/.github/workflows/install-dependencies-and-run-tests.yml b/.github/workflows/install-dependencies-and-run-tests.yml new file mode 100644 index 0000000..87572c0 --- /dev/null +++ b/.github/workflows/install-dependencies-and-run-tests.yml @@ -0,0 +1,42 @@ +name: Test Python package new version +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Update pip version + run: | + python -m pip install --upgrade pip + - name: Install dependencies + run: | + pip install -r test/requirements.txt + - name: Build new version of the package + run: | + python -m build + - name: Install the new version of the package + run: | + pip install dist/*.tar.gz + - name: Test with pytest + env: + HOSTNAME: ${{ secrets.HOSTNAME }} + PORT: ${{ secrets.PORT }} + ONTOLOGY: ${{ secrets.ONTOLOGY }} + USERNAME: ${{ secrets.USERNAME }} + PASSWORD: ${{ secrets.PASSWORD }} + ENABLED_SSL: ${{ secrets.ENABLED_SSL }} + HTTP_PATH: ${{ secrets.HTTP_PATH }} + run: | + pytest diff --git a/.gitignore b/.gitignore index efe0fda..5022aed 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +.vscode/launch.json # C extensions *.so @@ -50,7 +51,11 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ -test.py +test/env* +test/.python-version +test/.env +test/run* +test/*old* # Translations *.mo @@ -110,7 +115,6 @@ venv/ ENV/ env.bak/ venv.bak/ -.vscode # Spyder project settings .spyderproject diff --git a/README.md b/README.md index c9e0fdd..235019f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,12 @@ -![Timbr logo](https://timbr.ai/wp-content/uploads/2023/06/timbr-ai-l-5-226x60-1.png) - -![MIT License](https://img.shields.io/badge/License-MIT-green) -![GPL-2.0 License](https://img.shields.io/badge/License-GPL--2.0-green) -![Apache License 2.0](https://img.shields.io/badge/License-Apache%202.0-green) +![Timbr logo](https://timbr.ai/wp-content/uploads/2025/01/logotimbrai230125.png) [![FOSSA Status](https://app.fossa.com/api/projects/custom%2B50508%2Fgithub.com%2FWPSemantix%2Ftimbr_python_connector.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B50508%2Fgithub.com%2FWPSemantix%2Ftimbr_python_connector?ref=badge_shield&issueType=license) [![FOSSA Status](https://app.fossa.com/api/projects/custom%2B50508%2Fgithub.com%2FWPSemantix%2Ftimbr_python_connector.svg?type=shield&issueType=security)](https://app.fossa.com/projects/custom%2B50508%2Fgithub.com%2FWPSemantix%2Ftimbr_python_connector?ref=badge_shield&issueType=security) -[![Python 3.7.13](https://img.shields.io/badge/python-3.7.13+-blue)](https://www.python.org/downloads/release/python-3713/) -[![Python 3.8](https://img.shields.io/badge/python-3.8-blue)](https://www.python.org/downloads/release/python-3820/) [![Python 3.9](https://img.shields.io/badge/python-3.9-blue)](https://www.python.org/downloads/release/python-3921/) +[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-31017/) +[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-31112/) +[![Python 3.12](https://img.shields.io/badge/python-3.12-blue.svg)](https://www.python.org/downloads/release/python-3129/) [![Java 11](https://img.shields.io/badge/Java-11-red)](https://www.oracle.com/il-en/java/technologies/javase/jdk11-archive-downloads.html) [![Java 17](https://img.shields.io/badge/Java-17-red)](https://www.oracle.com/il-en/java/technologies/javase/jdk17-archive-downloads.html) @@ -21,12 +18,13 @@ This project is a python connector to timbr using JDBC. ## Dependencies -- Python 3.7.13+ or 3.8.x or 3.9.x +- Access to a timbr-server +- Python from 3.9.13 or newer - Java 11 or Java 17 or Java 21 ## Installation - Install as clone repository: - - Install Python: https://www.python.org/downloads/release/python-3713/ + - Install Python: https://www.python.org/downloads/release/python-3913/ - Install Java: https://www.oracle.com/il-en/java/technologies/javase/jdk11-archive-downloads.html - Run the following command to install the Python dependencies: `pip install -r requirements.txt` (optional install pandas to run pandas example) - Download the following jar to `jars` path: https://repo1.maven.org/maven2/org/apache/hive/hive-jdbc/4.0.1/hive-jdbc-4.0.1-standalone.jar diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8fe2f47 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/PyTimbr/__init__.py b/pytimbr/__init__.py similarity index 100% rename from PyTimbr/__init__.py rename to pytimbr/__init__.py diff --git a/PyTimbr/jars/hive-jdbc-4.0.1-standalone.jar b/pytimbr/jars/hive-jdbc-4.0.1-standalone.jar similarity index 100% rename from PyTimbr/jars/hive-jdbc-4.0.1-standalone.jar rename to pytimbr/jars/hive-jdbc-4.0.1-standalone.jar diff --git a/PyTimbr/timbr_connector.py b/pytimbr/timbr_connector.py similarity index 97% rename from PyTimbr/timbr_connector.py rename to pytimbr/timbr_connector.py index 107305b..b2828fc 100644 --- a/PyTimbr/timbr_connector.py +++ b/pytimbr/timbr_connector.py @@ -1,54 +1,54 @@ -# -# *### ., @% -# *%## `#// %%%* *@ `` @% -# #*. * .%%%` @@@@* @@ @@@@,@@@@ @&@@@@ .&@@@* -# #%%# .. *@ @@ @` @@` ,@ @% #@ @@ -# ,, .,%(##/./%%#, *@ @@ @` @@` ,@ @% #@ @@ -# ,%##% `` `/@@* @@ @` @@` ,@ (/@@@#/ @@ -# `` -# `````````````````````````````````````````````````````````````` -# Copyright (C) 2018-2024 timbr.ai -# - -from . import timbr_jdbapi -import os -import platform -import pathlib - -main_jar_path = os.environ.get('TIMBR_JDBC_JAR_PATH', os.path.join(pathlib.Path(__file__).parent.resolve(), 'jars')) -jdbc_driver = 'org.apache.hive.jdbc.HiveDriver' - -def get_combined_jars_path(maindir): - jar_dir = os.walk(maindir) - jars_array = [] - for _root, _dirs, files in jar_dir: - for filename in files: - if filename.find('.jar') > 0: - jars_array.append(os.path.join(maindir, filename)) - jars = ":".join(jars_array) - if "Windows" in platform.platform(): - jars = ";".join(jars_array) - - return jars - -jars_path = get_combined_jars_path(main_jar_path) - -def get_jdbc_connection(jdbc_url, username, password): - conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) - return conn - -def get_connection(hostname, port, ontology, username, password, enabled_ssl = 'true', http_path = '/timbr-server'): - jdbc_url = f"jdbc:hive2://{hostname}:{port}/{ontology};transportMode=http;ssl={enabled_ssl};httpPath={http_path}" - conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) - return conn - -# Deprecated - Backward compatibility - -def getJdbcConnection(jdbc_url, username, password): - conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) - return conn - -def getConnection(hostname, port, ontology, username, password, enabled_ssl = True, http_path = '/timbr-server'): - jdbc_url = f"jdbc:hive2://{hostname}:{port}/{ontology};transportMode=http;ssl={enabled_ssl};httpPath={http_path}" - conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) - return conn +# +# *### ., @% +# *%## `#// %%%* *@ `` @% +# #*. * .%%%` @@@@* @@ @@@@,@@@@ @&@@@@ .&@@@* +# #%%# .. *@ @@ @` @@` ,@ @% #@ @@ +# ,, .,%(##/./%%#, *@ @@ @` @@` ,@ @% #@ @@ +# ,%##% `` `/@@* @@ @` @@` ,@ (/@@@#/ @@ +# `` +# `````````````````````````````````````````````````````````````` +# Copyright (C) 2018-2024 timbr.ai +# + +from . import timbr_jdbapi +import os +import platform +import pathlib + +main_jar_path = os.environ.get('TIMBR_JDBC_JAR_PATH', os.path.join(pathlib.Path(__file__).parent.resolve(), 'jars')) +jdbc_driver = 'org.apache.hive.jdbc.HiveDriver' + +def get_combined_jars_path(maindir): + jar_dir = os.walk(maindir) + jars_array = [] + for _root, _dirs, files in jar_dir: + for filename in files: + if filename.find('.jar') > 0: + jars_array.append(os.path.join(maindir, filename)) + jars = ":".join(jars_array) + if "Windows" in platform.platform(): + jars = ";".join(jars_array) + + return jars + +jars_path = get_combined_jars_path(main_jar_path) + +def get_jdbc_connection(jdbc_url, username, password): + conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) + return conn + +def get_connection(hostname, port, ontology, username, password, enabled_ssl = 'true', http_path = '/timbr-server'): + jdbc_url = f"jdbc:hive2://{hostname}:{port}/{ontology};transportMode=http;ssl={enabled_ssl};httpPath={http_path}" + conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) + return conn + +# Deprecated - Backward compatibility + +def getJdbcConnection(jdbc_url, username, password): + conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) + return conn + +def getConnection(hostname, port, ontology, username, password, enabled_ssl = True, http_path = '/timbr-server'): + jdbc_url = f"jdbc:hive2://{hostname}:{port}/{ontology};transportMode=http;ssl={enabled_ssl};httpPath={http_path}" + conn = timbr_jdbapi.connect(jdbc_driver, jdbc_url, [username, password], jars_path) + return conn diff --git a/PyTimbr/timbr_jdbapi.py b/pytimbr/timbr_jdbapi.py similarity index 96% rename from PyTimbr/timbr_jdbapi.py rename to pytimbr/timbr_jdbapi.py index f26de44..f40b515 100644 --- a/PyTimbr/timbr_jdbapi.py +++ b/pytimbr/timbr_jdbapi.py @@ -1,722 +1,722 @@ -# Copyright 2010-2015 Bastian Bowe -# -# This file is part of JayDeBeApi. -# JayDeBeApi is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# JayDeBeApi is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with JayDeBeApi. If not, see -# . - -__version_info__ = (1, 2, 3) -__version__ = ".".join(str(i) for i in __version_info__) - -import datetime -import glob -import os -import time -import re -import sys -import warnings -import uuid -import jpype - -def reraise(tp, value, tb=None): - if value is None: - value = tp() - else: - value = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(value), flags=re.MULTILINE) - value = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", value, flags=re.MULTILINE) - value = value.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") - value = value.replace("java.util.concurrent.ExecutionException:", "").replace("Error running statement:", "") - value = Exception(value) - if tb: - raise value.with_traceback(tb) - raise value - -string_type = str - -# Mapping from java.sql.Types attribute name to attribute value -_jdbc_name_to_const = None - -# Mapping from java.sql.Types attribute constant value to it's attribute name -_jdbc_const_to_name = None - -_jdbc_connect = None - -_java_array_byte = None - -_handle_sql_exception = None - -old_jpype = False - -def _handle_sql_exception_jython(): - from java.sql import SQLException - exc_info = sys.exc_info() - if isinstance(exc_info[1], SQLException): - exc_type = DatabaseError - else: - exc_type = InterfaceError - reraise(exc_type, exc_info[1], exc_info[2]) - -def _jdbc_connect_jython(jclassname, url, driver_args, jars, libs): - if _jdbc_name_to_const is None: - from java.sql import Types - types = Types - types_map = {} - const_re = re.compile('[A-Z][A-Z_]*$') - for i in dir(types): - if const_re.match(i): - types_map[i] = getattr(types, i) - _init_types(types_map) - global _java_array_byte - if _java_array_byte is None: - import jarray - def _java_array_byte(data): - return jarray.array(data, 'b') - # register driver for DriverManager - jpackage = jclassname[:jclassname.rfind('.')] - dclassname = jclassname[jclassname.rfind('.') + 1:] - # print jpackage - # print dclassname - # print jpackage - from java.lang import Class - from java.lang import ClassNotFoundException - try: - Class.forName(jclassname).newInstance() - except ClassNotFoundException: - if not jars: - raise - _jython_set_classpath(jars) - Class.forName(jclassname).newInstance() - from java.sql import DriverManager - if isinstance(driver_args, dict): - from java.util import Properties - info = Properties() - for k, v in driver_args.items(): - info.setProperty(k, v) - dargs = [ info ] - else: - dargs = driver_args - return DriverManager.getConnection(url, *dargs) - -def _jython_set_classpath(jars): - ''' - import a jar at runtime (needed for JDBC [Class.forName]) - - adapted by Bastian Bowe from - http://stackoverflow.com/questions/3015059/jython-classpath-sys-path-and-jdbc-drivers - ''' - from java.net import URL, URLClassLoader - from java.lang import ClassLoader - from java.io import File - m = URLClassLoader.getDeclaredMethod("addURL", [URL]) - m.accessible = 1 - urls = [File(i).toURL() for i in jars] - m.invoke(ClassLoader.getSystemClassLoader(), urls) - -def _prepare_jython(): - global _jdbc_connect - _jdbc_connect = _jdbc_connect_jython - global _handle_sql_exception - _handle_sql_exception = _handle_sql_exception_jython - -def _handle_sql_exception_jpype(): - SQLException = jpype.java.sql.SQLException - exc_info = sys.exc_info() - if old_jpype: - clazz = exc_info[1].__javaclass__ - db_err = issubclass(clazz, SQLException) - else: - db_err = isinstance(exc_info[1], SQLException) - - if db_err: - exc_type = DatabaseError - else: - exc_type = InterfaceError - - reraise(exc_type, exc_info[1], exc_info[2]) - -def _jdbc_connect_jpype(jclassname, url, driver_args, jars, libs): - if not jpype.isJVMStarted(): - args = [] - class_path = [] - if jars: - class_path.extend(jars) - class_path.extend(_get_classpath()) - if class_path: - args.append('-Djava.class.path=%s' % - os.path.pathsep.join(class_path)) - if libs: - # path to shared libraries - libs_path = os.path.pathsep.join(libs) - args.append('-Djava.library.path=%s' % libs_path) - # jvm_path = ('/usr/lib/jvm/java-6-openjdk' - # '/jre/lib/i386/client/libjvm.so') - jvm_path = jpype.getDefaultJVMPath() - global old_jpype - if hasattr(jpype, '__version__'): - try: - ver_match = re.match('\\d+\\.\\d+', jpype.__version__) - if ver_match: - jpype_ver = float(ver_match.group(0)) - if jpype_ver < 0.7: - old_jpype = True - except ValueError: - pass - if old_jpype: - jpype.startJVM(jvm_path, *args) - else: - jpype.startJVM(jvm_path, *args, ignoreUnrecognized=True, - convertStrings=True) - if not jpype.java.lang.Thread.isAttached(): - jpype.attachThreadToJVM() - jpype.java.lang.Thread.currentThread().setContextClassLoader(jpype.java.lang.ClassLoader.getSystemClassLoader()) - global _jdbc_name_to_const - if _jdbc_name_to_const is None: - types = jpype.java.sql.Types - types_map = {} - if old_jpype: - for i in types.__javaclass__.getClassFields(): - const = i.getStaticAttribute() - types_map[i.getName()] = const - else: - for i in types.class_.getFields(): - if jpype.java.lang.reflect.Modifier.isStatic(i.getModifiers()): - const = i.get(None) - types_map[i.getName()] = const - _init_types(types_map) - global _java_array_byte - if _java_array_byte is None: - def _java_array_byte(data): - return jpype.JArray(jpype.JByte, 1)(data) - # register driver for DriverManager - jpype.JClass(jclassname) - - try: - response = jpype.java.sql.DriverManager.getConnection(url, driver_args[0], driver_args[1]) - except Exception as exception: - masked_exception = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(exception), flags=re.MULTILINE) - masked_exception = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", masked_exception, flags=re.MULTILINE) - masked_exception = masked_exception.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") - raise Exception(masked_exception) - - return response - -def _get_classpath(): - """Extract CLASSPATH from system environment as JPype doesn't seem - to respect that variable. - """ - try: - orig_cp = os.environ['CLASSPATH'] - except KeyError: - return [] - expanded_cp = [] - for i in orig_cp.split(os.path.pathsep): - expanded_cp.extend(_jar_glob(i)) - return expanded_cp - -def _jar_glob(item): - if item.endswith('*'): - return glob.glob('%s.[jJ][aA][rR]' % item) - else: - return [item] - -def _prepare_jpype(): - global _jdbc_connect - _jdbc_connect = _jdbc_connect_jpype - global _handle_sql_exception - _handle_sql_exception = _handle_sql_exception_jpype - -if sys.platform.lower().startswith('java'): - _prepare_jython() -else: - _prepare_jpype() - -apilevel = '2.0' -threadsafety = 1 -paramstyle = 'qmark' - -class DBAPITypeObject(object): - _mappings = {} - def __init__(self, *values): - """Construct new DB-API 2.0 type object. - values: Attribute names of java.sql.Types constants""" - self.values = values - for type_name in values: - if type_name in DBAPITypeObject._mappings: - raise ValueError("Non unique mapping for type '%s'" % type_name) - DBAPITypeObject._mappings[type_name] = self - def __cmp__(self, other): - if other in self.values: - return 0 - if other < self.values: - return 1 - else: - return -1 - def __repr__(self): - return 'DBAPITypeObject(%s)' % ", ".join([repr(i) for i in self.values]) - @classmethod - def _map_jdbc_type_to_dbapi(cls, jdbc_type_const): - try: - type_name = _jdbc_const_to_name[jdbc_type_const] - except KeyError: - warnings.warn("Unknown JDBC type with constant value %d. " - "Using None as a default type_code." % jdbc_type_const) - return None - try: - return cls._mappings[type_name] - except KeyError: - warnings.warn("No type mapping for JDBC type '%s' (constant value %d). " - "Using None as a default type_code." % (type_name, jdbc_type_const)) - return None - - -STRING = DBAPITypeObject('CHAR', 'NCHAR', 'NVARCHAR', 'VARCHAR', 'OTHER') - -TEXT = DBAPITypeObject('CLOB', 'LONGVARCHAR', 'LONGNVARCHAR', 'NCLOB', 'SQLXML') - -BINARY = DBAPITypeObject('BINARY', 'BLOB', 'LONGVARBINARY', 'VARBINARY') - -NUMBER = DBAPITypeObject('BOOLEAN', 'BIGINT', 'BIT', 'INTEGER', 'SMALLINT', - 'TINYINT') - -FLOAT = DBAPITypeObject('FLOAT', 'REAL', 'DOUBLE') - -DECIMAL = DBAPITypeObject('DECIMAL', 'NUMERIC') - -DATE = DBAPITypeObject('DATE') - -TIME = DBAPITypeObject('TIME') - -DATETIME = DBAPITypeObject('TIMESTAMP') - -ROWID = DBAPITypeObject('ROWID') - -# DB-API 2.0 Module Interface Exceptions -class Error(Exception): - pass - -class Warning(Exception): - pass - -class InterfaceError(Error): - pass - -class DatabaseError(Error): - pass - -class InternalError(DatabaseError): - pass - -class OperationalError(DatabaseError): - pass - -class ProgrammingError(DatabaseError): - pass - -class IntegrityError(DatabaseError): - pass - -class DataError(DatabaseError): - pass - -class NotSupportedError(DatabaseError): - pass - -# DB-API 2.0 Type Objects and Constructors - -def _java_sql_blob(data): - return _java_array_byte(data) - -Binary = _java_sql_blob - -def _str_func(func): - def to_str(*parms): - return str(func(*parms)) - return to_str - -Date = _str_func(datetime.date) - -Time = _str_func(datetime.time) - -Timestamp = _str_func(datetime.datetime) - -def DateFromTicks(ticks): - return apply(Date, time.localtime(ticks)[:3]) - -def TimeFromTicks(ticks): - return apply(Time, time.localtime(ticks)[3:6]) - -def TimestampFromTicks(ticks): - return apply(Timestamp, time.localtime(ticks)[:6]) - -# DB-API 2.0 Module Interface connect constructor -def connect(jclassname, url, driver_args=None, jars=None, libs=None): - """Open a connection to a database using a JDBC driver and return - a Connection instance. - - jclassname: Full qualified Java class name of the JDBC driver. - url: Database url as required by the JDBC driver. - driver_args: Dictionary or sequence of arguments to be passed to - the Java DriverManager.getConnection method. Usually - sequence of username and password for the db. Alternatively - a dictionary of connection arguments (where `user` and - `password` would probably be included). See - http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html - for more details - jars: Jar filename or sequence of filenames for the JDBC driver - libs: Dll/so filenames or sequence of dlls/sos used as shared - library by the JDBC driver - """ - if isinstance(driver_args, string_type): - driver_args = [ driver_args ] - if not driver_args: - driver_args = [] - if jars: - if isinstance(jars, string_type): - jars = [ jars ] - else: - jars = [] - if libs: - if isinstance(libs, string_type): - libs = [ libs ] - else: - libs = [] - try: - jconn = _jdbc_connect(jclassname, url, driver_args, jars, libs) - response = Connection(jconn, _converters) - except Exception as exception: - masked_exception = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(exception), flags=re.MULTILINE) - masked_exception = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", masked_exception, flags=re.MULTILINE) - masked_exception = masked_exception.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") - raise Exception(masked_exception) - - return response - -# DB-API 2.0 Connection Object -class Connection(object): - - Error = Error - Warning = Warning - InterfaceError = InterfaceError - DatabaseError = DatabaseError - InternalError = InternalError - OperationalError = OperationalError - ProgrammingError = ProgrammingError - IntegrityError = IntegrityError - DataError = DataError - NotSupportedError = NotSupportedError - parse_boolean_string = False - - def __init__(self, jconn, converters): - self.jconn = jconn - self._closed = False - self._converters = converters - - def close(self): - if self._closed: - raise Error() - self.jconn.close() - self._closed = True - - def commit(self): - try: - self.jconn.commit() - except: - _handle_sql_exception() - - def rollback(self): - try: - self.jconn.rollback() - except: - _handle_sql_exception() - - def cursor(self): - return Cursor(self, self._converters) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - -# DB-API 2.0 Cursor Object -class Cursor(object): - - rowcount = -1 - _meta = None - _prep = None - _rs = None - _description = None - - def __init__(self, connection, converters): - self._connection = connection - self._converters = converters - - @property - def description(self): - if self._description: - return self._description - m = self._meta - if m: - count = m.getColumnCount() - self._description = [] - for col in range(1, count + 1): - size = m.getColumnDisplaySize(col) - jdbc_type = m.getColumnType(col) - if jdbc_type == 0: - # PEP-0249: SQL NULL values are represented by the - # Python None singleton - dbapi_type = None - else: - dbapi_type = DBAPITypeObject._map_jdbc_type_to_dbapi(jdbc_type) - col_desc = ( m.getColumnName(col), - dbapi_type, - size, - size, - m.getPrecision(col), - m.getScale(col), - m.isNullable(col), - ) - self._description.append(col_desc) - return self._description - -# optional callproc(self, procname, *parameters) unsupported - - def close(self): - self._close_last() - self._connection = None - - def _close_last(self): - """Close the resultset and reset collected meta data. - """ - if self._rs: - self._rs.close() - self._rs = None - if self._prep: - self._prep.close() - self._prep = None - self._meta = None - self._description = None - - def _set_stmt_parms(self, prep_stmt, parameters): - for i in range(len(parameters)): - # print (i, parameters[i], type(parameters[i])) - prep_stmt.setObject(i + 1, parameters[i]) - - def execute(self, operation, parameters=None): - if self._connection._closed: - raise Error() - if not parameters: - parameters = () - self._close_last() - self._prep = self._connection.jconn.prepareStatement(operation) - self._set_stmt_parms(self._prep, parameters) - try: - is_rs = self._prep.execute() - except: - _handle_sql_exception() - if is_rs: - self._rs = self._prep.getResultSet() - self._meta = self._rs.getMetaData() - self.rowcount = -1 - else: - self.rowcount = self._prep.getUpdateCount() - # self._prep.getWarnings() ??? - - def executemany(self, operation, seq_of_parameters): - self._close_last() - self._prep = self._connection.jconn.prepareStatement(operation) - for parameters in seq_of_parameters: - self._set_stmt_parms(self._prep, parameters) - self._prep.addBatch() - update_counts = self._prep.executeBatch() - # self._prep.getWarnings() ??? - self.rowcount = sum(update_counts) - self._close_last() - - def fetchone(self): - if not self._rs: - raise Error() - if not self._rs.next(): - return None - row = [] - for col in range(1, self._meta.getColumnCount() + 1): - sqltype = self._meta.getColumnType(col) - converter = self._converters.get(sqltype) - - if converter is not None: - v = converter(self._connection, self._rs, col, sqltype) - else: - v = _java_to_py(self._connection, self._rs, col, 12) # DEFAULT GET VARCHAR - - row.append(v) - return row - - def fetchmany(self, size=None): - if not self._rs: - raise Error() - if size is None: - size = self.arraysize - # TODO: handle SQLException if not supported by db - self._rs.setFetchSize(size) - rows = [] - row = None - for i in range(size): - row = self.fetchone() - if row is None: - break - else: - rows.append(row) - # reset fetch size - if row: - # TODO: handle SQLException if not supported by db - self._rs.setFetchSize(0) - self._rs.close() - return rows - - def fetchall(self): - rows = [] - while True: - row = self.fetchone() - if row is None: - break - else: - rows.append(row) - self._rs.close() - return rows - - # optional nextset() unsupported - - arraysize = 1 - - def setinputsizes(self, sizes): - pass - - def setoutputsize(self, size, column=None): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - -def _to_datetime(conn, rs, col, sqltype): - java_val = rs.getTimestamp(col) - if not java_val or rs.wasNull(): - return - d = datetime.datetime.strptime(str(java_val)[:19], "%Y-%m-%d %H:%M:%S") - d = d.replace(microsecond=int(str(java_val.getNanos())[:6])) - return str(d) - -def _to_time(conn, rs, col, sqltype): - java_val = rs.getTime(col) - if not java_val or rs.wasNull(): - return - return str(java_val) - -def _to_date(conn, rs, col, sqltype): - java_val = rs.getDate(col) - if not java_val or rs.wasNull(): - return - # The following code requires Python 3.3+ on dates before year 1900. - # d = datetime.datetime.strptime(str(java_val)[:10], "%Y-%m-%d") - # return d.strftime("%Y-%m-%d") - # Workaround / simpler soltution (see - # https://github.com/baztian/jaydebeapi/issues/18): - return str(java_val)[:10] - -def _java_to_py(conn, rs, col, sqltype): - - if sqltype == 12 or sqltype == 0: # STRING the most common type in the start - v = rs.getString(col) - elif sqltype == -7 or sqltype == 16: # BOOLEAN - v = rs.getBoolean(col) - - if conn and not conn.parse_boolean_string: - if v: - v = 1 - else: - v = 0 - - elif sqltype == -6: # TINYINT - v = rs.getByte(col) - elif sqltype == 5: # SMALLINT - v = rs.getShort(col) - elif sqltype == 4: # INTEGER - v = rs.getInt(col) - elif sqltype == -5: # BIGINT - v = rs.getLong(col) - elif sqltype == 8 or sqltype == 6: # DOUBLE OR FLOAT - v = rs.getDouble(col) - else: # For all other types return string - v = rs.getString(col) - - if rs.wasNull(): - return - - return v - -def _java_to_py_bigdecimal(conn, rs, col, sqltype): - java_val = rs.getBigDecimal(col) - - if java_val is None or rs.wasNull(): - return - if hasattr(java_val, 'scale'): - scale = java_val.scale() - if scale == 0: - return java_val.longValue() - else: - return java_val.doubleValue() - else: - return float(java_val) - -def _init_types(types_map): - global _jdbc_name_to_const - _jdbc_name_to_const = types_map - global _jdbc_const_to_name - _jdbc_const_to_name = dict((y,x) for x,y in types_map.items()) - _init_converters(types_map) - -def _init_converters(types_map): - """Prepares the converters for conversion of java types to python - objects. - types_map: Mapping of java.sql.Types field name to java.sql.Types - field constant value""" - global _converters - _converters = {} - for i in _DEFAULT_CONVERTERS: - const_val = types_map[i] - _converters[const_val] = _DEFAULT_CONVERTERS[i] - -# Mapping from java.sql.Types field to converter method -_converters = None - -_DEFAULT_CONVERTERS = { - # see - # http://download.oracle.com/javase/8/docs/api/java/sql/Types.html - # for possible keys - 'TIMESTAMP': _to_datetime, - 'TIME': _to_time, - 'DATE': _to_date, - 'DECIMAL': _java_to_py_bigdecimal, - 'NUMERIC': _java_to_py_bigdecimal, - 'BOOLEAN': _java_to_py, - 'BIT': _java_to_py, - 'DOUBLE': _java_to_py, - 'FLOAT': _java_to_py, - 'TINYINT': _java_to_py, - 'INTEGER': _java_to_py, - 'BIGINT': _java_to_py, - 'SMALLINT': _java_to_py, - 'VARCHAR': _java_to_py -} +# Copyright 2010-2015 Bastian Bowe +# +# This file is part of JayDeBeApi. +# JayDeBeApi is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# JayDeBeApi is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with JayDeBeApi. If not, see +# . + +__version_info__ = (1, 2, 3) +__version__ = ".".join(str(i) for i in __version_info__) + +import datetime +import glob +import os +import time +import re +import sys +import warnings +import uuid +import jpype + +def reraise(tp, value, tb=None): + if value is None: + value = tp() + else: + value = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(value), flags=re.MULTILINE) + value = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", value, flags=re.MULTILINE) + value = value.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") + value = value.replace("java.util.concurrent.ExecutionException:", "").replace("Error running statement:", "") + value = Exception(value) + if tb: + raise value.with_traceback(tb) + raise value + +string_type = str + +# Mapping from java.sql.Types attribute name to attribute value +_jdbc_name_to_const = None + +# Mapping from java.sql.Types attribute constant value to it's attribute name +_jdbc_const_to_name = None + +_jdbc_connect = None + +_java_array_byte = None + +_handle_sql_exception = None + +old_jpype = False + +def _handle_sql_exception_jython(): + from java.sql import SQLException + exc_info = sys.exc_info() + if isinstance(exc_info[1], SQLException): + exc_type = DatabaseError + else: + exc_type = InterfaceError + reraise(exc_type, exc_info[1], exc_info[2]) + +def _jdbc_connect_jython(jclassname, url, driver_args, jars, libs): + if _jdbc_name_to_const is None: + from java.sql import Types + types = Types + types_map = {} + const_re = re.compile('[A-Z][A-Z_]*$') + for i in dir(types): + if const_re.match(i): + types_map[i] = getattr(types, i) + _init_types(types_map) + global _java_array_byte + if _java_array_byte is None: + import jarray + def _java_array_byte(data): + return jarray.array(data, 'b') + # register driver for DriverManager + jpackage = jclassname[:jclassname.rfind('.')] + dclassname = jclassname[jclassname.rfind('.') + 1:] + # print jpackage + # print dclassname + # print jpackage + from java.lang import Class + from java.lang import ClassNotFoundException + try: + Class.forName(jclassname).newInstance() + except ClassNotFoundException: + if not jars: + raise + _jython_set_classpath(jars) + Class.forName(jclassname).newInstance() + from java.sql import DriverManager + if isinstance(driver_args, dict): + from java.util import Properties + info = Properties() + for k, v in driver_args.items(): + info.setProperty(k, v) + dargs = [ info ] + else: + dargs = driver_args + return DriverManager.getConnection(url, *dargs) + +def _jython_set_classpath(jars): + ''' + import a jar at runtime (needed for JDBC [Class.forName]) + + adapted by Bastian Bowe from + http://stackoverflow.com/questions/3015059/jython-classpath-sys-path-and-jdbc-drivers + ''' + from java.net import URL, URLClassLoader + from java.lang import ClassLoader + from java.io import File + m = URLClassLoader.getDeclaredMethod("addURL", [URL]) + m.accessible = 1 + urls = [File(i).toURL() for i in jars] + m.invoke(ClassLoader.getSystemClassLoader(), urls) + +def _prepare_jython(): + global _jdbc_connect + _jdbc_connect = _jdbc_connect_jython + global _handle_sql_exception + _handle_sql_exception = _handle_sql_exception_jython + +def _handle_sql_exception_jpype(): + SQLException = jpype.java.sql.SQLException + exc_info = sys.exc_info() + if old_jpype: + clazz = exc_info[1].__javaclass__ + db_err = issubclass(clazz, SQLException) + else: + db_err = isinstance(exc_info[1], SQLException) + + if db_err: + exc_type = DatabaseError + else: + exc_type = InterfaceError + + reraise(exc_type, exc_info[1], exc_info[2]) + +def _jdbc_connect_jpype(jclassname, url, driver_args, jars, libs): + if not jpype.isJVMStarted(): + args = [] + class_path = [] + if jars: + class_path.extend(jars) + class_path.extend(_get_classpath()) + if class_path: + args.append('-Djava.class.path=%s' % + os.path.pathsep.join(class_path)) + if libs: + # path to shared libraries + libs_path = os.path.pathsep.join(libs) + args.append('-Djava.library.path=%s' % libs_path) + # jvm_path = ('/usr/lib/jvm/java-6-openjdk' + # '/jre/lib/i386/client/libjvm.so') + jvm_path = jpype.getDefaultJVMPath() + global old_jpype + if hasattr(jpype, '__version__'): + try: + ver_match = re.match('\\d+\\.\\d+', jpype.__version__) + if ver_match: + jpype_ver = float(ver_match.group(0)) + if jpype_ver < 0.7: + old_jpype = True + except ValueError: + pass + if old_jpype: + jpype.startJVM(jvm_path, *args) + else: + jpype.startJVM(jvm_path, *args, ignoreUnrecognized=True, + convertStrings=True) + if not jpype.java.lang.Thread.isAttached(): + jpype.attachThreadToJVM() + jpype.java.lang.Thread.currentThread().setContextClassLoader(jpype.java.lang.ClassLoader.getSystemClassLoader()) + global _jdbc_name_to_const + if _jdbc_name_to_const is None: + types = jpype.java.sql.Types + types_map = {} + if old_jpype: + for i in types.__javaclass__.getClassFields(): + const = i.getStaticAttribute() + types_map[i.getName()] = const + else: + for i in types.class_.getFields(): + if jpype.java.lang.reflect.Modifier.isStatic(i.getModifiers()): + const = i.get(None) + types_map[i.getName()] = const + _init_types(types_map) + global _java_array_byte + if _java_array_byte is None: + def _java_array_byte(data): + return jpype.JArray(jpype.JByte, 1)(data) + # register driver for DriverManager + jpype.JClass(jclassname) + + try: + response = jpype.java.sql.DriverManager.getConnection(url, driver_args[0], driver_args[1]) + except Exception as exception: + masked_exception = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(exception), flags=re.MULTILINE) + masked_exception = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", masked_exception, flags=re.MULTILINE) + masked_exception = masked_exception.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") + raise Exception(masked_exception) + + return response + +def _get_classpath(): + """Extract CLASSPATH from system environment as JPype doesn't seem + to respect that variable. + """ + try: + orig_cp = os.environ['CLASSPATH'] + except KeyError: + return [] + expanded_cp = [] + for i in orig_cp.split(os.path.pathsep): + expanded_cp.extend(_jar_glob(i)) + return expanded_cp + +def _jar_glob(item): + if item.endswith('*'): + return glob.glob('%s.[jJ][aA][rR]' % item) + else: + return [item] + +def _prepare_jpype(): + global _jdbc_connect + _jdbc_connect = _jdbc_connect_jpype + global _handle_sql_exception + _handle_sql_exception = _handle_sql_exception_jpype + +if sys.platform.lower().startswith('java'): + _prepare_jython() +else: + _prepare_jpype() + +apilevel = '2.0' +threadsafety = 1 +paramstyle = 'qmark' + +class DBAPITypeObject(object): + _mappings = {} + def __init__(self, *values): + """Construct new DB-API 2.0 type object. + values: Attribute names of java.sql.Types constants""" + self.values = values + for type_name in values: + if type_name in DBAPITypeObject._mappings: + raise ValueError("Non unique mapping for type '%s'" % type_name) + DBAPITypeObject._mappings[type_name] = self + def __cmp__(self, other): + if other in self.values: + return 0 + if other < self.values: + return 1 + else: + return -1 + def __repr__(self): + return 'DBAPITypeObject(%s)' % ", ".join([repr(i) for i in self.values]) + @classmethod + def _map_jdbc_type_to_dbapi(cls, jdbc_type_const): + try: + type_name = _jdbc_const_to_name[jdbc_type_const] + except KeyError: + warnings.warn("Unknown JDBC type with constant value %d. " + "Using None as a default type_code." % jdbc_type_const) + return None + try: + return cls._mappings[type_name] + except KeyError: + warnings.warn("No type mapping for JDBC type '%s' (constant value %d). " + "Using None as a default type_code." % (type_name, jdbc_type_const)) + return None + + +STRING = DBAPITypeObject('CHAR', 'NCHAR', 'NVARCHAR', 'VARCHAR', 'OTHER') + +TEXT = DBAPITypeObject('CLOB', 'LONGVARCHAR', 'LONGNVARCHAR', 'NCLOB', 'SQLXML') + +BINARY = DBAPITypeObject('BINARY', 'BLOB', 'LONGVARBINARY', 'VARBINARY') + +NUMBER = DBAPITypeObject('BOOLEAN', 'BIGINT', 'BIT', 'INTEGER', 'SMALLINT', + 'TINYINT') + +FLOAT = DBAPITypeObject('FLOAT', 'REAL', 'DOUBLE') + +DECIMAL = DBAPITypeObject('DECIMAL', 'NUMERIC') + +DATE = DBAPITypeObject('DATE') + +TIME = DBAPITypeObject('TIME') + +DATETIME = DBAPITypeObject('TIMESTAMP') + +ROWID = DBAPITypeObject('ROWID') + +# DB-API 2.0 Module Interface Exceptions +class Error(Exception): + pass + +class Warning(Exception): + pass + +class InterfaceError(Error): + pass + +class DatabaseError(Error): + pass + +class InternalError(DatabaseError): + pass + +class OperationalError(DatabaseError): + pass + +class ProgrammingError(DatabaseError): + pass + +class IntegrityError(DatabaseError): + pass + +class DataError(DatabaseError): + pass + +class NotSupportedError(DatabaseError): + pass + +# DB-API 2.0 Type Objects and Constructors + +def _java_sql_blob(data): + return _java_array_byte(data) + +Binary = _java_sql_blob + +def _str_func(func): + def to_str(*parms): + return str(func(*parms)) + return to_str + +Date = _str_func(datetime.date) + +Time = _str_func(datetime.time) + +Timestamp = _str_func(datetime.datetime) + +def DateFromTicks(ticks): + return apply(Date, time.localtime(ticks)[:3]) + +def TimeFromTicks(ticks): + return apply(Time, time.localtime(ticks)[3:6]) + +def TimestampFromTicks(ticks): + return apply(Timestamp, time.localtime(ticks)[:6]) + +# DB-API 2.0 Module Interface connect constructor +def connect(jclassname, url, driver_args=None, jars=None, libs=None): + """Open a connection to a database using a JDBC driver and return + a Connection instance. + + jclassname: Full qualified Java class name of the JDBC driver. + url: Database url as required by the JDBC driver. + driver_args: Dictionary or sequence of arguments to be passed to + the Java DriverManager.getConnection method. Usually + sequence of username and password for the db. Alternatively + a dictionary of connection arguments (where `user` and + `password` would probably be included). See + http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html + for more details + jars: Jar filename or sequence of filenames for the JDBC driver + libs: Dll/so filenames or sequence of dlls/sos used as shared + library by the JDBC driver + """ + if isinstance(driver_args, string_type): + driver_args = [ driver_args ] + if not driver_args: + driver_args = [] + if jars: + if isinstance(jars, string_type): + jars = [ jars ] + else: + jars = [] + if libs: + if isinstance(libs, string_type): + libs = [ libs ] + else: + libs = [] + try: + jconn = _jdbc_connect(jclassname, url, driver_args, jars, libs) + response = Connection(jconn, _converters) + except Exception as exception: + masked_exception = re.sub('user=(.*);password=(.*);', 'user=XXXXXXXXXX;password=XXXXXXXXXX;', str(exception), flags=re.MULTILINE) + masked_exception = re.sub("USER '(.*)', PASSWORD '(.*)'", "USER 'XXXXXXXXXX', PASSWORD 'XXXXXXXXXX'", masked_exception, flags=re.MULTILINE) + masked_exception = masked_exception.replace("org.apache.hive.service.cli.HiveSQLException:", "").replace("java.sql.SQLException:", "").replace("java.lang.RuntimeException:", "") + raise Exception(masked_exception) + + return response + +# DB-API 2.0 Connection Object +class Connection(object): + + Error = Error + Warning = Warning + InterfaceError = InterfaceError + DatabaseError = DatabaseError + InternalError = InternalError + OperationalError = OperationalError + ProgrammingError = ProgrammingError + IntegrityError = IntegrityError + DataError = DataError + NotSupportedError = NotSupportedError + parse_boolean_string = False + + def __init__(self, jconn, converters): + self.jconn = jconn + self._closed = False + self._converters = converters + + def close(self): + if self._closed: + raise Error() + self.jconn.close() + self._closed = True + + def commit(self): + try: + self.jconn.commit() + except: + _handle_sql_exception() + + def rollback(self): + try: + self.jconn.rollback() + except: + _handle_sql_exception() + + def cursor(self): + return Cursor(self, self._converters) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + +# DB-API 2.0 Cursor Object +class Cursor(object): + + rowcount = -1 + _meta = None + _prep = None + _rs = None + _description = None + + def __init__(self, connection, converters): + self._connection = connection + self._converters = converters + + @property + def description(self): + if self._description: + return self._description + m = self._meta + if m: + count = m.getColumnCount() + self._description = [] + for col in range(1, count + 1): + size = m.getColumnDisplaySize(col) + jdbc_type = m.getColumnType(col) + if jdbc_type == 0: + # PEP-0249: SQL NULL values are represented by the + # Python None singleton + dbapi_type = None + else: + dbapi_type = DBAPITypeObject._map_jdbc_type_to_dbapi(jdbc_type) + col_desc = ( m.getColumnName(col), + dbapi_type, + size, + size, + m.getPrecision(col), + m.getScale(col), + m.isNullable(col), + ) + self._description.append(col_desc) + return self._description + +# optional callproc(self, procname, *parameters) unsupported + + def close(self): + self._close_last() + self._connection = None + + def _close_last(self): + """Close the resultset and reset collected meta data. + """ + if self._rs: + self._rs.close() + self._rs = None + if self._prep: + self._prep.close() + self._prep = None + self._meta = None + self._description = None + + def _set_stmt_parms(self, prep_stmt, parameters): + for i in range(len(parameters)): + # print (i, parameters[i], type(parameters[i])) + prep_stmt.setObject(i + 1, parameters[i]) + + def execute(self, operation, parameters=None): + if self._connection._closed: + raise Error() + if not parameters: + parameters = () + self._close_last() + self._prep = self._connection.jconn.prepareStatement(operation) + self._set_stmt_parms(self._prep, parameters) + try: + is_rs = self._prep.execute() + except: + _handle_sql_exception() + if is_rs: + self._rs = self._prep.getResultSet() + self._meta = self._rs.getMetaData() + self.rowcount = -1 + else: + self.rowcount = self._prep.getUpdateCount() + # self._prep.getWarnings() ??? + + def executemany(self, operation, seq_of_parameters): + self._close_last() + self._prep = self._connection.jconn.prepareStatement(operation) + for parameters in seq_of_parameters: + self._set_stmt_parms(self._prep, parameters) + self._prep.addBatch() + update_counts = self._prep.executeBatch() + # self._prep.getWarnings() ??? + self.rowcount = sum(update_counts) + self._close_last() + + def fetchone(self): + if not self._rs: + raise Error() + if not self._rs.next(): + return None + row = [] + for col in range(1, self._meta.getColumnCount() + 1): + sqltype = self._meta.getColumnType(col) + converter = self._converters.get(sqltype) + + if converter is not None: + v = converter(self._connection, self._rs, col, sqltype) + else: + v = _java_to_py(self._connection, self._rs, col, 12) # DEFAULT GET VARCHAR + + row.append(v) + return row + + def fetchmany(self, size=None): + if not self._rs: + raise Error() + if size is None: + size = self.arraysize + # TODO: handle SQLException if not supported by db + self._rs.setFetchSize(size) + rows = [] + row = None + for i in range(size): + row = self.fetchone() + if row is None: + break + else: + rows.append(row) + # reset fetch size + if row: + # TODO: handle SQLException if not supported by db + self._rs.setFetchSize(0) + self._rs.close() + return rows + + def fetchall(self): + rows = [] + while True: + row = self.fetchone() + if row is None: + break + else: + rows.append(row) + self._rs.close() + return rows + + # optional nextset() unsupported + + arraysize = 1 + + def setinputsizes(self, sizes): + pass + + def setoutputsize(self, size, column=None): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + +def _to_datetime(conn, rs, col, sqltype): + java_val = rs.getTimestamp(col) + if not java_val or rs.wasNull(): + return + d = datetime.datetime.strptime(str(java_val)[:19], "%Y-%m-%d %H:%M:%S") + d = d.replace(microsecond=int(str(java_val.getNanos())[:6])) + return str(d) + +def _to_time(conn, rs, col, sqltype): + java_val = rs.getTime(col) + if not java_val or rs.wasNull(): + return + return str(java_val) + +def _to_date(conn, rs, col, sqltype): + java_val = rs.getDate(col) + if not java_val or rs.wasNull(): + return + # The following code requires Python 3.3+ on dates before year 1900. + # d = datetime.datetime.strptime(str(java_val)[:10], "%Y-%m-%d") + # return d.strftime("%Y-%m-%d") + # Workaround / simpler soltution (see + # https://github.com/baztian/jaydebeapi/issues/18): + return str(java_val)[:10] + +def _java_to_py(conn, rs, col, sqltype): + + if sqltype == 12 or sqltype == 0: # STRING the most common type in the start + v = rs.getString(col) + elif sqltype == -7 or sqltype == 16: # BOOLEAN + v = rs.getBoolean(col) + + if conn and not conn.parse_boolean_string: + if v: + v = 1 + else: + v = 0 + + elif sqltype == -6: # TINYINT + v = rs.getByte(col) + elif sqltype == 5: # SMALLINT + v = rs.getShort(col) + elif sqltype == 4: # INTEGER + v = rs.getInt(col) + elif sqltype == -5: # BIGINT + v = rs.getLong(col) + elif sqltype == 8 or sqltype == 6: # DOUBLE OR FLOAT + v = rs.getDouble(col) + else: # For all other types return string + v = rs.getString(col) + + if rs.wasNull(): + return + + return v + +def _java_to_py_bigdecimal(conn, rs, col, sqltype): + java_val = rs.getBigDecimal(col) + + if java_val is None or rs.wasNull(): + return + if hasattr(java_val, 'scale'): + scale = java_val.scale() + if scale == 0: + return java_val.longValue() + else: + return java_val.doubleValue() + else: + return float(java_val) + +def _init_types(types_map): + global _jdbc_name_to_const + _jdbc_name_to_const = types_map + global _jdbc_const_to_name + _jdbc_const_to_name = dict((y,x) for x,y in types_map.items()) + _init_converters(types_map) + +def _init_converters(types_map): + """Prepares the converters for conversion of java types to python + objects. + types_map: Mapping of java.sql.Types field name to java.sql.Types + field constant value""" + global _converters + _converters = {} + for i in _DEFAULT_CONVERTERS: + const_val = types_map[i] + _converters[const_val] = _DEFAULT_CONVERTERS[i] + +# Mapping from java.sql.Types field to converter method +_converters = None + +_DEFAULT_CONVERTERS = { + # see + # http://download.oracle.com/javase/8/docs/api/java/sql/Types.html + # for possible keys + 'TIMESTAMP': _to_datetime, + 'TIME': _to_time, + 'DATE': _to_date, + 'DECIMAL': _java_to_py_bigdecimal, + 'NUMERIC': _java_to_py_bigdecimal, + 'BOOLEAN': _java_to_py, + 'BIT': _java_to_py, + 'DOUBLE': _java_to_py, + 'FLOAT': _java_to_py, + 'TINYINT': _java_to_py, + 'INTEGER': _java_to_py, + 'BIGINT': _java_to_py, + 'SMALLINT': _java_to_py, + 'VARCHAR': _java_to_py +} diff --git a/requirements.txt b/requirements.txt index c24f9c9..df415f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ JPype1==1.5.1 -# pandas==1.3.5 -# numpy==1.26.4 \ No newline at end of file +pandas==1.3.5 +numpy==1.26.4 \ No newline at end of file diff --git a/setup.py b/setup.py index 147907d..5a05269 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,8 @@ packages=['pytimbr'], install_requires=[ 'JPype1==1.5.1', + 'pandas==1.3.5', + 'numpy==1.26.4', ], package_data={ 'pytimbr': ['jars/*'], diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..861a6c0 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,19 @@ +import os +import pytest +from dotenv import load_dotenv + +# Load .env file if it exists +load_dotenv(override=True) + +# Global fixture to load config values +@pytest.fixture(scope="session") +def test_config(): + return { + "hostname": os.getenv("HOSTNAME"), + "port": os.getenv("PORT"), + "ontology": os.getenv("ONTOLOGY"), + "username": os.getenv("USERNAME"), + "password": os.getenv("PASSWORD"), + "enabled_ssl": os.getenv("ENABLED_SSL"), + "http_path": os.getenv("HTTP_PATH"), + } diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..80c1251 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,6 @@ +python-dotenv>=1.1.0 +pytest>=8.3.4 +build>=1.2 +JPype1==1.5.1 +pandas==1.3.5 +numpy==1.26.4 \ No newline at end of file diff --git a/test/test_connection.py b/test/test_connection.py new file mode 100644 index 0000000..05ef405 --- /dev/null +++ b/test/test_connection.py @@ -0,0 +1,37 @@ +import pytest +import pytimbr as timbr +from utils import get_connection_uri, parse_and_print_results + +def test_basic_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.get_connection(hostname, port, ontology, username, password, enabled_ssl, http_path) + result_obj = parse_and_print_results(conn) + + assert conn is not None, "Connection object is None." + assert result_obj["column_count"] > 0, "No columns found in the cursor metadata." + assert len(result_obj["description"]) > 0, "No columns found in the cursor description." + assert len(result_obj["concepts"]) > 0, "No concepts found in the database." + +def test_jdbc_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.get_jdbc_connection(get_connection_uri(hostname, port, ontology, enabled_ssl, http_path), username, password) + result_obj = parse_and_print_results(conn) + + assert conn is not None, "Connection object is None." + assert result_obj["column_count"] > 0, "No columns found in the cursor metadata." + assert len(result_obj["description"]) > 0, "No columns found in the cursor description." + assert len(result_obj["concepts"]) > 0, "No concepts found in the database." \ No newline at end of file diff --git a/test/test_connection_using_pandas.py b/test/test_connection_using_pandas.py new file mode 100644 index 0000000..5cf9754 --- /dev/null +++ b/test/test_connection_using_pandas.py @@ -0,0 +1,36 @@ +import pytest +import pytimbr as timbr +import pandas as pd +from utils import get_connection_uri, parse_and_print_results_using_pandas + +def test_basic_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.get_connection(hostname, port, ontology, username, password, enabled_ssl, http_path) + + df = parse_and_print_results_using_pandas(conn) + + assert not df.empty, "DataFrame is empty." + assert len(df.columns) > 0, "No columns found in the DataFrame." + +def test_jdbc_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.get_jdbc_connection(get_connection_uri(hostname, port, ontology, enabled_ssl, http_path), username, password) + + df = parse_and_print_results_using_pandas(conn) + + assert not df.empty, "DataFrame is empty." + assert len(df.columns) > 0, "No columns found in the DataFrame." \ No newline at end of file diff --git a/test/test_legacy_connection.py b/test/test_legacy_connection.py new file mode 100644 index 0000000..ad4d3c7 --- /dev/null +++ b/test/test_legacy_connection.py @@ -0,0 +1,37 @@ +import pytest +import pytimbr as timbr +from utils import get_connection_uri, parse_and_print_results + +def test_basic_legacy_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.getConnection(hostname, port, ontology, username, password, enabled_ssl, http_path) + result_obj = parse_and_print_results(conn) + + assert conn is not None, "Connection object is None." + assert result_obj["column_count"] > 0, "No columns found in the cursor metadata." + assert len(result_obj["description"]) > 0, "No columns found in the cursor description." + assert len(result_obj["concepts"]) > 0, "No concepts found in the database." + +def test_jdbc_legacy_connection(test_config): + hostname = test_config['hostname'] + port = test_config['port'] + ontology = test_config['ontology'] + username = test_config['username'] + password = test_config['password'] + enabled_ssl = test_config['enabled_ssl'] + http_path = test_config['http_path'] + + conn = timbr.getJdbcConnection(get_connection_uri(hostname,port,ontology,enabled_ssl,http_path), username, password) + result_obj = parse_and_print_results(conn) + + assert conn is not None, "Connection object is None." + assert result_obj["column_count"] > 0, "No columns found in the cursor metadata." + assert len(result_obj["description"]) > 0, "No columns found in the cursor description." + assert len(result_obj["concepts"]) > 0, "No concepts found in the database." \ No newline at end of file diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000..26e8264 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,60 @@ +import pandas as pd + +def get_connection_uri(hostname: str, port: int, ontology: str, enabled_ssl: str, http_path: str) -> str: + """ + Constructs a connection URI for the Hive database using the provided parameters. + + :param hostname: The hostname of the Hive server. + :param port: The port number on which the Hive server is listening. + :param ontology: The ontology or database name. + :param enabled_ssl: A flag indicating whether SSL is enabled (e.g., 'true' or 'false'). + :param http_path: The HTTP path for the Hive server. + + :return: A formatted connection URI string for Hive. + """ + return f"jdbc:hive2://{hostname}:{port}/{ontology};transportMode=http;ssl={enabled_ssl};httpPath={http_path}" + +def parse_and_print_results(connection): + """ + Parses and prints the results from the database cursor. + :param connection: The database connection object. + """ + + with connection.cursor() as curs: + curs.execute('SHOW CONCEPTS') + concepts = curs.fetchall() + + # Recommended + for i in range(1, curs._meta.getColumnCount() + 1): + print(curs._meta.getColumnName(i) + " - " + curs._meta.getColumnTypeName(i)) + + # DBAPI + for col in curs.description: + print(col[0] + " - " + col[1].values[0]) + + # Print the results + for concept in concepts: + print(concept) + + return dict( + column_count=curs._meta.getColumnCount(), + description=curs.description, + concepts=concepts, + ) + +def parse_and_print_results_using_pandas(connection): + """ + Parses and prints the results from the database cursor using pandas. + :param connection: The database connection object. + """ + + with connection.cursor() as curs: + df = pd.read_sql('SHOW CONCEPTS', connection) + print("--------------------------------------") + print(df) + print("--------------------------------------") + print(df.columns) + print("--------------------------------------") + print(df.count()) + + return df \ No newline at end of file