GH-46734: [C++] Arrow Flight SQL ODBC layer

.github/workflows/cpp_extra.yml

@@ -336,14 +336,81 @@ jobs:
           cd cpp/examples/minimal_build
           ../minimal_build.build/arrow-example
 
-  odbc:
+  odbc-macos:
     needs: check-labels
-    name: ODBC
-    runs-on: windows-2022
+    name: ODBC ${{ matrix.architecture }} macOS ${{ matrix.macos-version }}
+    runs-on: macos-${{ matrix.macos-version }}
     if: >-
       needs.check-labels.outputs.force == 'true' ||
       contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
       contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - architecture: AMD64
+            macos-version: "15-intel"
+          - architecture: ARM64
+            macos-version: "14"
+    env:
+      ARROW_BUILD_TESTS: ON
+      ARROW_FLIGHT_SQL_ODBC: ON
+      ARROW_HOME: /tmp/local
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v6.0.0
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Install Dependencies
+        run: |
+          brew bundle --file=cpp/Brewfile
+      - name: Setup ccache
+        run: |
+          ci/scripts/ccache_setup.sh
+      - name: ccache info
+        id: ccache-info
+        run: |
+          echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
+      - name: Cache ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.ccache-info.outputs.cache-dir }}
+          key: cpp-ccache-macos-${{ matrix.macos-version }}-${{ hashFiles('cpp/**') }}
+          restore-keys: cpp-ccache-macos-${{ matrix.macos-version }}-
+      - name: Build
+        run: |
+          # Homebrew uses /usr/local as prefix. So packages
+          # installed by Homebrew also use /usr/local/include. We
+          # want to include headers for packages installed by
+          # Homebrew as system headers to ignore warnings in them.
+          # But "-isystem /usr/local/include" isn't used by CMake
+          # because /usr/local/include is marked as the default
+          # include path. So we disable -Werror to avoid build error
+          # by warnings from packages installed by Homebrew.
+          export BUILD_WARNING_LEVEL=PRODUCTION
+          LIBIODBC_DIR="$(brew --cellar libiodbc)/$(brew list --versions libiodbc | awk '{print $2}')"
+          export ODBC_INCLUDE_DIR=$LIBIODBC_DIR/include
+          export CXXFLAGS="$CXXFLAGS -I$ODBC_INCLUDE_DIR"
+          ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
+      - name: Test
+        shell: bash
+        run: |
+          sudo sysctl -w kern.coredump=1
+          sudo sysctl -w kern.corefile=/tmp/core.%N.%P
+          ulimit -c unlimited  # must enable within the same shell
+          ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
+
+  odbc-msvc:
+    needs: check-labels
+    name: ODBC Windows
+    runs-on: windows-2022
+    if: >-
+      needs.check-labels.outputs.force == 'true' ||
+      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
+      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') ||
+      contains(join(github.event.pull_request.changed_files, ' '), 'cpp/src/arrow/flight/sql/odbc/')
     timeout-minutes: 240
     permissions:
       packages: write
@@ -465,6 +532,7 @@ jobs:
       - jni-linux
       - jni-macos
       - msvc-arm64
-      - odbc
+      - odbc-macos
+      - odbc-msvc
     uses: ./.github/workflows/report_ci.yml
     secrets: inherit

.gitignore

@@ -32,6 +32,7 @@ dependency-reduced-pom.xml
 MANIFEST
 compile_commands.json
 build.ninja
+build*/
 
 # Generated Visual Studio files
 *.vcxproj

ci/scripts/cpp_build.sh

@@ -272,6 +272,7 @@ else
     -DgRPC_SOURCE=${gRPC_SOURCE:-} \
     -DGTest_SOURCE=${GTest_SOURCE:-} \
     -Dlz4_SOURCE=${lz4_SOURCE:-} \
+    -DODBC_INCLUDE_DIR="${ODBC_INCLUDE_DIR:-}" \
     -Dopentelemetry-cpp_SOURCE=${opentelemetry_cpp_SOURCE:-} \
     -DORC_SOURCE=${ORC_SOURCE:-} \
     -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \

ci/scripts/cpp_test.sh

@@ -59,6 +59,7 @@ case "$(uname)" in
     ;;
   Darwin)
     n_jobs=$(sysctl -n hw.ncpu)
+    exclude_tests+=("arrow-flight-sql-odbc-test")
     # TODO: https://github.com/apache/arrow/issues/40410
     exclude_tests+=("arrow-s3fs-test")
     ;;

cpp/Brewfile

@@ -28,6 +28,7 @@ brew "git"
 brew "glog"
 brew "googletest"
 brew "grpc"
+brew "libiodbc"
 brew "llvm"
 brew "lz4"
 brew "mimalloc"

cpp/CMakePresets.json

@@ -315,6 +315,17 @@
       "displayName": "Debug build with tests and Flight SQL",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-debug-flight-sql-odbc",
+      "inherits": [
+        "features-flight-sql",
+        "base-debug"
+      ],
+      "displayName": "Debug build with tests and Flight SQL ODBC",
+      "cacheVariables": {
+        "ARROW_FLIGHT_SQL_ODBC": "ON"
+      }
+    },
     {
       "name": "ninja-debug-gandiva",
       "inherits": [
@@ -511,6 +522,17 @@
       "displayName": "Release build with Flight SQL",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-release-flight-sql-odbc",
+      "inherits": [
+        "features-flight-sql",
+        "base-release"
+      ],
+      "displayName": "Release build with Flight SQL ODBC",
+      "cacheVariables": {
+        "ARROW_FLIGHT_SQL_ODBC": "ON"
+      }
+    },
     {
       "name": "ninja-release-gandiva",
       "inherits": [

cpp/cmake_modules/DefineOptions.cmake

@@ -107,8 +107,8 @@ macro(tsort_bool_option_dependencies)
 endmacro()
 
 macro(resolve_option_dependencies)
-  # Arrow Flight SQL ODBC is available only for Windows for now.
-  if(NOT WIN32)
+  # Arrow Flight SQL ODBC is available only for Windows and macOS for now.
+  if(NOT WIN32 AND NOT APPLE)
     set(ARROW_FLIGHT_SQL_ODBC OFF)
   endif()
   if(MSVC_TOOLCHAIN)

cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc

@@ -65,33 +65,43 @@ Status SqliteTablesWithSchemaBatchReader::ReadNext(std::shared_ptr<RecordBatch>*
 
   auto* string_array = reinterpret_cast<StringArray*>(table_name_array.get());
 
-  std::vector<std::shared_ptr<Field>> column_fields;
+  std::map<std::string, std::vector<std::shared_ptr<Field>>> table_columns_map;
   for (int i = 0; i < table_name_array->length(); i++) {
     const std::string& table_name = string_array->GetString(i);
+    table_columns_map[table_name];
+  }
 
-    while (sqlite3_step(schema_statement->GetSqlite3Stmt()) == SQLITE_ROW) {
-      std::string sqlite_table_name = std::string(reinterpret_cast<const char*>(
-          sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 0)));
-      if (sqlite_table_name == table_name) {
-        const char* column_name = reinterpret_cast<const char*>(
-            sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 1));
-        const char* column_type = reinterpret_cast<const char*>(
-            sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 2));
-        int nullable = sqlite3_column_int(schema_statement->GetSqlite3Stmt(), 3);
-
-        const ColumnMetadata& column_metadata = GetColumnMetadata(
-            GetSqlTypeFromTypeName(column_type), sqlite_table_name.c_str());
-        std::shared_ptr<DataType> arrow_type;
-        auto status = GetArrowType(column_type).Value(&arrow_type);
-        if (!status.ok()) {
-          return Status::NotImplemented("Unknown SQLite type '", column_type,
-                                        "' for column '", column_name, "' in table '",
-                                        table_name, "': ", status);
-        }
-        column_fields.push_back(arrow::field(column_name, arrow_type, nullable == 0,
-                                             column_metadata.metadata_map()));
+  while (sqlite3_step(schema_statement->GetSqlite3Stmt()) == SQLITE_ROW) {
+    std::string table_name = std::string(reinterpret_cast<const char*>(
+        sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 0)));
+
+    if (table_columns_map.contains(table_name)) {
+      const char* column_name = reinterpret_cast<const char*>(
+          sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 1));
+      const char* column_type = reinterpret_cast<const char*>(
+          sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 2));
+      int nullable = sqlite3_column_int(schema_statement->GetSqlite3Stmt(), 3);
+
+      const ColumnMetadata& column_metadata =
+          GetColumnMetadata(GetSqlTypeFromTypeName(column_type), table_name.c_str());
+
+      std::shared_ptr<DataType> arrow_type;
+      auto status = GetArrowType(column_type).Value(&arrow_type);
+      if (!status.ok()) {
+        return Status::NotImplemented("Unknown SQLite type '", column_type,
+                                      "' for column '", column_name, "' in table '",
+                                      table_name, "': ", status);
       }
+      table_columns_map[table_name].push_back(arrow::field(
+          column_name, arrow_type, nullable == 0, column_metadata.metadata_map()));
     }
+  }
+
+  std::vector<std::shared_ptr<Field>> column_fields;
+  for (int i = 0; i < table_name_array->length(); i++) {
+    const std::string& table_name = string_array->GetString(i);
+    column_fields = table_columns_map[table_name];
+
     ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Buffer> schema_buffer,
                           ipc::SerializeSchema(*arrow::schema(column_fields)));
 

cpp/src/arrow/flight/sql/odbc/README

@@ -0,0 +1,83 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+## Steps to Register the 64-bit Apache Arrow ODBC driver on Windows
+
+After the build succeeds, the ODBC DLL will be located in 
+`build\debug\Debug` for a debug build and `build\release\Release` for a release build.
+
+1. Open Power Shell as administrator.
+
+2. Register your ODBC DLL:
+    Need to replace <path\to\repo> with actual path to repository in the commands.
+
+    i. `cd to repo.`
+    ii. `cd <path\to\repo>`
+    iii. Run script to register your ODBC DLL as Apache Arrow Flight SQL ODBC Driver
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd <path\to\repo>\cpp\build\< release | debug >\< Release | Debug>\arrow_flight_sql_odbc.dll`
+    Example command for reference:
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd C:\path\to\arrow\cpp\build\release\Release\arrow_flight_sql_odbc.dll`
+
+If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
+should show as an available ODBC driver in the x64 ODBC Driver Manager.
+
+## Steps to Generate Windows Installer
+1. Build with  `ARROW_FLIGHT_SQL_ODBC=ON` and `ARROW_FLIGHT_SQL_ODBC_INSTALLER=ON`.
+2. `cd` to `build` folder.
+3. Run `cpack`. 
+
+If the generation is successful, you will find `Apache Arrow Flight SQL ODBC-<version>-win64.msi` generated under the `build` folder.
+
+## Steps to Register the 64-bit Apache Arrow ODBC driver on macOS
+
+After the build succeeds, the ODBC DYLIB will be located in 
+`build\debug` for a debug build and `build\release` for a release build.
+
+1. Open terminal shell.
+
+2. Register your ODBC DYLIB:
+    Need to replace <path\to\repo> with actual path to repository in the commands.
+
+    i. `cd to repo.`
+    ii. `cd <path\to\repo>`
+    iii. Give script permission to execute
+    `chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh`
+    iv. Run script with `sudo` to register your ODBC DYLIB as Apache Arrow Flight SQL ODBC Driver
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh <path\to\repo>/cpp/build/< release | debug >/libarrow_flight_sql_odbc.dylib`
+    Example command for reference:
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh /path/to/arrow/cpp/build/release/libarrow_flight_sql_odbc.dylib`
+
+If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
+should be shown at `~/Library/ODBC/odbcinst.ini`
+
+## Steps to Enable Logging
+Arrow Flight SQL ODBC driver uses Arrow's internal logging framework. By default, the log messages are printed to the terminal.
+1. Set environment variable `ARROW_ODBC_LOG_LEVEL` to any of the following valid values to enable logging. If `ARROW_ODBC_LOG_LEVEL` is set to a non-empty string that does not match any of the following values, `DEBUG` level is used by default. 
+
+The characters are case-insensitive.
+- TRACE
+- DEBUG 
+- INFO
+- WARNING
+- ERROR 
+- FATAL
+
+The Windows ODBC driver currently does not support writing log files. `ARROW_USE_GLOG` is required to write log files, and `ARROW_USE_GLOG` is disabled on Windows platform since plasma using `glog` is not fully tested on windows.
+
+Note: GH-47670 running more than 1 tests with logging enabled is not fully supported. 

cpp/src/arrow/flight/sql/odbc/entry_points.cc

@@ -83,6 +83,18 @@ SQLRETURN SQL_API SQLGetDiagRec(SQLSMALLINT handle_type, SQLHANDLE handle,
       buffer_length, text_length_ptr);
 }
 
+#if defined(__APPLE__)
+// macOS ODBC Driver Manager doesn't map SQLError to SQLGetDiagRec, so we need to
+// implement SQLError for macOS.
+// on Windows, SQLError mapping implemented by Driver Manager is preferred.
+SQLRETURN SQL_API SQLError(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, SQLWCHAR* sql_state,
+                           SQLINTEGER* native_error_ptr, SQLWCHAR* message_text,
+                           SQLSMALLINT buffer_length, SQLSMALLINT* text_length_ptr) {
+  return arrow::flight::sql::odbc::SQLError(env, conn, stmt, sql_state, native_error_ptr,
+                                            message_text, buffer_length, text_length_ptr);
+}
+#endif  // __APPLE__
+
 SQLRETURN SQL_API SQLGetEnvAttr(SQLHENV env, SQLINTEGER attr, SQLPOINTER value_ptr,
                                 SQLINTEGER buffer_len, SQLINTEGER* str_len_ptr) {
   return arrow::flight::sql::odbc::SQLGetEnvAttr(env, attr, value_ptr, buffer_len,