diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 110061c59..ca96df524 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 ARG POSTGRES_INSTALL_ARG= -ARG PG_VERSION=16 +ARG IVORY_VERSION=4 ARG CITUS_VERSION=12 # declare installed PG version and Citus version -ENV PG_VERSION=${PG_VERSION} +ENV IVORY_VERSION=${IVORY_VERSION} ENV CITUS_VERSION=${CITUS_VERSION} # Install build essentials - Compiler, debugger, make, etc. @@ -35,6 +35,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libtool \ libicu-dev \ libssl-dev \ + libxml2-dev \ && rm -rf /var/lib/apt/lists/* # Add pgdg repo @@ -60,15 +61,15 @@ COPY scripts/utils.sh /tmp/install_setup/ # Install postgres COPY scripts/install_setup_postgres.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/usr/lib/postgresql/${PG_VERSION}\" $POSTGRES_INSTALL_ARG -v ${PG_VERSION}" ] +RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/var/local/ivorysql/ivory-${IVORY_VERSION}\" $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION}" ] # Install RUM from source COPY scripts/install_setup_rum_oss.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ] # Install citus COPY scripts/install_setup_citus_core_oss.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ] # Install citus-indent COPY scripts/install_citus_indent.sh /tmp/install_setup/ @@ -76,11 +77,11 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_citus_indent.sh" ] # Install SYSTEM_ROWS COPY scripts/install_setup_system_rows.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh" ] # Install PG_CRON COPY scripts/install_setup_pg_cron.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ] # Download Decimal128 Intel library COPY scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/ @@ -92,7 +93,7 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_pcre2.sh" ] # Install PG_VECTOR COPY scripts/install_setup_pgvector.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh" ] # Install PostGIS from source RUN apt-get update && \ @@ -105,7 +106,7 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* COPY scripts/install_setup_postgis.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh" ] # locale RUN rm -rf /var/lib/apt/lists/* \ @@ -121,10 +122,10 @@ RUN echo "%sudo ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/no-pass-ask # Add postgres to path for sudo commands. # Add path for sudoers -RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/usr\/lib\/postgresql\/$PG_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path +RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/var\/local\/ivorysql\/ivory-$IVORY_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path # Add PG to the path -ENV PATH=$PATH:/usr/lib/postgresql/$PG_VERSION/bin +ENV PATH=$PATH:/var/local/ivorysql/ivory-$IVORY_VERSION/bin USER documentdb WORKDIR /home/documentdb diff --git a/.github/containers/Build-Ubuntu/Dockerfile b/.github/containers/Build-Ubuntu/Dockerfile index db2d9494c..ca96df524 100644 --- a/.github/containers/Build-Ubuntu/Dockerfile +++ b/.github/containers/Build-Ubuntu/Dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 ARG POSTGRES_INSTALL_ARG= -ARG PG_VERSION=16 +ARG IVORY_VERSION=4 ARG CITUS_VERSION=12 # declare installed PG version and Citus version -ENV PG_VERSION=${PG_VERSION} +ENV IVORY_VERSION=${IVORY_VERSION} ENV CITUS_VERSION=${CITUS_VERSION} # Install build essentials - Compiler, debugger, make, etc. @@ -20,8 +20,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ pkg-config \ cmake \ git \ - locales \ - gcc \ + locales \ + gcc \ gdb \ libipc-run-perl \ unzip \ @@ -35,6 +35,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libtool \ libicu-dev \ libssl-dev \ + libxml2-dev \ && rm -rf /var/lib/apt/lists/* # Add pgdg repo @@ -60,15 +61,15 @@ COPY scripts/utils.sh /tmp/install_setup/ # Install postgres COPY scripts/install_setup_postgres.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/usr/lib/postgresql/${PG_VERSION}\" $POSTGRES_INSTALL_ARG -v ${PG_VERSION}" ] +RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/var/local/ivorysql/ivory-${IVORY_VERSION}\" $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION}" ] # Install RUM from source COPY scripts/install_setup_rum_oss.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ] # Install citus COPY scripts/install_setup_citus_core_oss.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ] # Install citus-indent COPY scripts/install_citus_indent.sh /tmp/install_setup/ @@ -76,11 +77,11 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_citus_indent.sh" ] # Install SYSTEM_ROWS COPY scripts/install_setup_system_rows.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh" ] # Install PG_CRON COPY scripts/install_setup_pg_cron.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ] # Download Decimal128 Intel library COPY scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/ @@ -92,19 +93,20 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_pcre2.sh" ] # Install PG_VECTOR COPY scripts/install_setup_pgvector.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh" ] # Install PostGIS from source -RUN add-apt-repository -y ppa:ubuntugis/ppa && apt-get update && \ +RUN apt-get update && \ apt-get install -qy \ libproj-dev \ libxml2-dev \ libjson-c-dev \ - libgeos-dev \ libgeos++-dev \ + libgeos-dev \ && rm -rf /var/lib/apt/lists/* + COPY scripts/install_setup_postgis.sh /tmp/install_setup/ -RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh" ] +RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh" ] # locale RUN rm -rf /var/lib/apt/lists/* \ @@ -120,14 +122,10 @@ RUN echo "%sudo ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/no-pass-ask # Add postgres to path for sudo commands. # Add path for sudoers -RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/usr\/lib\/postgresql\/$PG_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path +RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/var\/local\/ivorysql\/ivory-$IVORY_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path # Add PG to the path -ENV PATH=$PATH:/usr/lib/postgresql/$PG_VERSION/bin +ENV PATH=$PATH:/var/local/ivorysql/ivory-$IVORY_VERSION/bin USER documentdb WORKDIR /home/documentdb - -LABEL org.opencontainers.image.source=https://github.com/microsoft/documentdb -LABEL org.opencontainers.image.description="DocumentDB ubuntu build image" -LABEL org.opencontainers.image.licenses=MIT \ No newline at end of file diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml index 297261b6f..ee95fdd51 100644 --- a/.github/workflows/build_packages.yml +++ b/.github/workflows/build_packages.yml @@ -38,9 +38,8 @@ jobs: runner: ubuntu-24.04 - arch: arm64 runner: ubuntu-24.04-arm - pg_version: - - 16 - - 17 + ivory_version: + - 4 steps: - name: Checkout repository @@ -56,12 +55,12 @@ jobs: - name: Build Debian Package run: | - ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg_version }} --version ${{ env.DOCUMENTDB_VERSION }} + ./packaging/build_packages.sh --os ${{ matrix.os }} --ivy ${{ matrix.ivory_version }} --version ${{ env.DOCUMENTDB_VERSION }} - name: Upload Debian Package as Artifact uses: actions/upload-artifact@v4 with: - name: archive-${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }} + name: archive-${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.ivory_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }} path: | packaging/*.deb retention-days: 7 @@ -121,10 +120,10 @@ jobs: push: true platforms: linux/${{ matrix.arch }} file: .github/containers/Build-Ubuntu/Dockerfile_prebuild - tags: ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} + tags: ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} build-args: | BASE_IMAGE=${{ env.BASE_IMAGE }} - POSTGRES_VERSION=${{ matrix.pg_version }} + IVORY_VERSION=${{ matrix.ivory_version }} DEB_PACKAGE_REL_PATH=${{ env.PACKAGE_NAME }} labels: | org.opencontainers.image.source=https://github.com/microsoft/documentdb @@ -138,7 +137,7 @@ jobs: - name: Sign manifest (keyless) if: startsWith(github.ref, 'refs/tags/') run: | - cosign sign ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} -y + cosign sign ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} -y - name: Verify manifest signature (keyless) if: startsWith(github.ref, 'refs/tags/') @@ -147,4 +146,4 @@ jobs: cosign verify \ --certificate-identity-regexp "https://github.com/${{ github.repository }}/.github/workflows/build_packages.yml@refs/(heads|tags)/${{ github.ref_name }}" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ - ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} + ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} diff --git a/.github/workflows/build_rpm_packages.yml b/.github/workflows/build_rpm_packages.yml index 717bd76c4..96ad36710 100644 --- a/.github/workflows/build_rpm_packages.yml +++ b/.github/workflows/build_rpm_packages.yml @@ -32,9 +32,8 @@ jobs: include: - arch: amd64 runner: ubuntu-24.04 - pg_version: - - 16 - - 17 + ivory_version: + - 4 steps: - name: Checkout repository @@ -50,12 +49,12 @@ jobs: - name: Build RPM Package run: | - ./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg_version }} --version ${{ env.DOCUMENTDB_VERSION }} --test-clean-install + ./packaging/build_packages.sh --os ${{ matrix.os }} --ivy ${{ matrix.ivory_version }} --version ${{ env.DOCUMENTDB_VERSION }} --test-clean-install - name: Upload RPM Package as Artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }} + name: ${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.ivory_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }} path: | packaging/*.rpm retention-days: 7 diff --git a/.github/workflows/regress_tests.yml b/.github/workflows/regress_tests.yml index aba9404d8..945027f79 100644 --- a/.github/workflows/regress_tests.yml +++ b/.github/workflows/regress_tests.yml @@ -27,10 +27,8 @@ jobs: strategy: fail-fast: false matrix: - pg_version: - - 15 - - 16 - - 17 + ivory_version: + - 4 arch: - amd64 - arm64 @@ -51,7 +49,7 @@ jobs: - name: Install package dependencies run: | - export PG_VERSION=${{ matrix.pg_version }} + export IVORY_VERSION=${{ matrix.ivory_version }} export CITUS_VERSION=12 export LC_ALL=en_US.UTF-8 @@ -100,23 +98,23 @@ jobs: sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT MAKE_PROGRAM=cmake /tmp/install_setup/install_setup_libbson.sh cp ./scripts/utils.sh /tmp/install_setup cp ./scripts/install_setup_postgres.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_postgres.sh -d /usr/lib/postgresql/${PG_VERSION} $POSTGRES_INSTALL_ARG -v ${PG_VERSION} + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_postgres.sh -d /var/local/ivorysql/ivory-${IVORY_VERSION} $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION} cp ./scripts/install_setup_rum_oss.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh cp ./scripts/install_setup_citus_core_oss.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION} + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION} cp ./scripts/install_citus_indent.sh /tmp/install_setup/ sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_citus_indent.sh cp ./scripts/install_setup_system_rows.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh cp ./scripts/install_setup_pg_cron.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh cp ./scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/ sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_intel_decimal_math_lib.sh cp ./scripts/install_setup_pcre2.sh /tmp/install_setup/ sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_pcre2.sh cp ./scripts/install_setup_pgvector.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh sudo apt-get update sudo apt-get install -qy \ @@ -128,17 +126,17 @@ jobs: libgeos-dev \ cp ./scripts/install_setup_postgis.sh /tmp/install_setup/ - sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh + sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh - name: Export pg_config PATH run: | - echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH + echo "/usr/lib/postgresql/${{ matrix.ivory_version }}/bin" >> $GITHUB_PATH - name: Configure, Build and Install DocumentDB run: | which pg_config - make - sudo PATH=$PATH make install + env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make + sudo env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make install - name: Run Tests run: | @@ -147,11 +145,11 @@ jobs: export LANGUAGE=en_US export LC_COLLATE=en_US.UTF-8 export LC_CTYPE=en_US.UTF-8 - make check + env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make check - name: Citus Indent run: | - citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1) + env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1) - name: Validate sanity of files run: | @@ -161,6 +159,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs_$${{ matrix.runner }}_PG${{ matrix.pg_version }} + name: logs_$${{ matrix.runner }}_PG${{ matrix.ivory_version }} overwrite: true path: "**/*.log" diff --git a/README.md b/README.md index 86b2a5a2d..c982ff4ca 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,19 @@ # Introduction -`DocumentDB` is a MongoDB compatible open source document database built on PostgreSQL. It offers a native implementation of a document-oriented NoSQL database, enabling seamless CRUD (Create, Read, Update, Delete) operations on BSON(Binary JSON) data types within a PostgreSQL framework. Beyond basic operations, DocumentDB empowers users to execute complex workloads, including full-text searches, geospatial queries, and vector search, delivering robust functionality and flexibility for diverse data management needs. +`ivydocumentdb` is an open-source project developed based on Microsoft DocumentDB and compatible with IvorySQL. It offers a native implementation of document-oriented NoSQL database, enabling seamless CRUD (Create, Read, Update, Delete) operations on BSON(Binary JSON) data types within an IvorySQL framework. Beyond basic operations, ivydocumentdb empowers you to execute complex workloads, including full-text searches, geospatial queries, and vector embeddings on your dataset, delivering robust functionality and flexibility for diverse data management needs. + +[IvorySQL](https://docs.ivorysql.org/en/ivorysql-doc) is advanced, fully featured, open source Oracle compatible PostgreSQL with a firm commitment to always remain 100% compatible and a Drop-in replacement of the latest PostgreSQL. [PostgreSQL](https://www.postgresql.org/about/) is a powerful, open source object-relational database system that uses and extends the SQL language combined with many features that safely store and scale the most complicated data workloads. +[DocumentDB](https://github.com/documentdb/documentdb) is the engine powering vCore-based Azure Cosmos DB for MongoDB. + ## Components -The project comprises of three components, which work together to support document operations. +The project comprises of two primary components, which work together to support document operations. - **pg_documentdb_core :** PostgreSQL extension introducing BSON datatype support and operations for native Postgres. - **pg_documentdb :** The public API surface for DocumentDB providing CRUD functionality on documents in the store. -- **pg_documentdb_gw :** The gateway protocol translation layer that converts the user's MongoDB APIs into PostgreSQL queries. ## Why DocumentDB ? @@ -31,140 +34,238 @@ We chose PostgreSQL as our platform for several reasons: ## Get Started -### Prerequisites -- Python 3.7+ -- pip package manager -- Docker -- Git (for cloning the repository) +### Pre-requisite + +- Ensure [Docker](https://docs.docker.com/engine/install/) is installed on your system. -Step 1: Install Python +### Building DocumentDB with Docker + +Step 1: Clone the ivydocumentdb repo. ```bash +git clone https://github.com/ivorysql/ivydocumentdb.git +``` -pip install pymongo +Step 2: Create the docker image. Navigate to cloned repo. +```bash +docker build . -f .devcontainer/Dockerfile -t ivydocumentdb ``` -Step 2. Install optional dependencies +Note: Validate using `docker image ls` + +Step 3: Run the Image as a container ```bash +docker run -v $(pwd):/home/documentdb/code -it ivydocumentdb /bin/bash + +cd code +``` -pip install dnspython +(Aligns local location with docker image created, allows de-duplicating cloning repo again within image).
+Note: Validate container is running `docker container ls` +Step 4: Build & Deploy the binaries + +```bash +make ``` -Step 3. Setup DocumentDB using Docker +Note: Run in case of an unsuccessful build `git config --global --add safe.directory /home/documentdb/code` within image. ```bash +sudo make install +``` + +Note: To run backend postgresql tests after installing you can run `make check`. - # Pull the latest DocumentDB Docker image - docker pull ghcr.io/microsoft/documentdb/documentdb-local:latest +You are all set to work with DocumentDB. + +### Connecting to the Server +#### Internal Access +Step 1: Run `start_oss_server.sh` to initialize the DocumentDB server and manage dependencies. + +```bash +./scripts/start_oss_server.sh +``` - # Tag the image for convenience - docker tag ghcr.io/microsoft/documentdb/documentdb-local:latest documentdb +Or logging into the container if using prebuild image +```bash +docker exec -it bash +``` - # Run the container with your chosen username and password - docker run -dt -p 10260:10260 --name documentdb-container documentdb --username --password - docker image rm -f ghcr.io/microsoft/documentdb/documentdb-local:latest || echo "No existing documentdb image to remove" +Step 2: Connect to `psql` shell +```bash +psql -p 9712 -d postgres ``` -> **Note:** During the transition to the Linux Foundation, Docker images may still be hosted on Microsoft's container registry. These will be migrated to the new DocumentDB organization as the transition completes. - > **Note:** Replace `` and `` with your desired credentials. You must set these when creating the container for authentication to work. - > - > **Port Note:** Port `10260` is used by default in these instructions to avoid conflicts with other local database services. You can use port `27017` (the standard MongoDB port) or any other available port if you prefer. If you do, be sure to update the port number in both your `docker run` command and your connection string accordingly. +#### External Access +Connect to `psql` shell -Step 4: Initialize the pymongo client with the credentials from the previous step +```bash +psql -h localhost --port 9712 -d postgres -U documentdb +``` -```python +## Usage -import pymongo +Once you have your `DocumentDB` set up running, you can start with creating collections, indexes and perform queries on them. -from pymongo import MongoClient +### Create a collection -# Create a MongoDB client and open a connection to DocumentDB -client = pymongo.MongoClient( - 'mongodb://:@localhost:10260/?tls=true&tlsAllowInvalidCertificates=true' -) +DocumentDB provides [documentdb_api.create_collection](https://github.com/microsoft/documentdb/wiki/Functions#create_collection) function to create a new collection within a specified database, enabling you to manage and organize your BSON documents effectively. +```sql +SELECT documentdb_api.create_collection('documentdb','patient'); ``` -Step 5: Create a database and collection +### Perform CRUD operations -```python +#### Insert documents -quickStartDatabase = client["quickStartDatabase"] -quickStartCollection = quickStartDatabase.create_collection("quickStartCollection") +The [documentdb_api.insert_one](https://github.com/microsoft/documentdb/wiki/Functions#insert_one) command is used to add a single document into a collection. +```sql +select documentdb_api.insert_one('documentdb','patient', '{ "patient_id": "P001", "name": "Alice Smith", "age": 30, "phone_number": "555-0123", "registration_year": "2023","conditions": ["Diabetes", "Hypertension"]}'); +select documentdb_api.insert_one('documentdb','patient', '{ "patient_id": "P002", "name": "Bob Johnson", "age": 45, "phone_number": "555-0456", "registration_year": "2023", "conditions": ["Asthma"]}'); +select documentdb_api.insert_one('documentdb','patient', '{ "patient_id": "P003", "name": "Charlie Brown", "age": 29, "phone_number": "555-0789", "registration_year": "2024", "conditions": ["Allergy", "Anemia"]}'); +select documentdb_api.insert_one('documentdb','patient', '{ "patient_id": "P004", "name": "Diana Prince", "age": 40, "phone_number": "555-0987", "registration_year": "2024", "conditions": ["Migraine"]}'); +select documentdb_api.insert_one('documentdb','patient', '{ "patient_id": "P005", "name": "Edward Norton", "age": 55, "phone_number": "555-1111", "registration_year": "2025", "conditions": ["Hypertension", "Heart Disease"]}'); ``` -Step 6: Insert documents +#### Read document from a collection -```python +The `documentdb_api.collection` function is used for retrieving the documents in a collection. -# Insert a single document -quickStartCollection.insert_one({ - 'name': 'John Doe', - 'email': 'john@email.com', - 'address': '123 Main St, Anytown, USA', - 'phone': '555-1234' - }) +```sql +SELECT document FROM documentdb_api.collection('documentdb','patient'); +``` + +Alternatively, we can apply filter to our queries. -# Insert multiple documents -quickStartCollection.insert_many([ - { - 'name': 'Jane Smith', - 'email': 'jane@email.com', - 'address': '456 Elm St, Othertown, USA', - 'phone': '555-5678' - }, - { - 'name': 'Alice Johnson', - 'email': 'alice@email.com', - 'address': '789 Oak St, Sometown, USA', - 'phone': '555-8765' - } -]) +```sql +SET search_path TO documentdb_api, documentdb_core; +SET documentdb_core.bsonUseEJson TO true; +SELECT cursorPage FROM documentdb_api.find_cursor_first_page('documentdb', '{ "find" : "patient", "filter" : {"patient_id":"P005"}}'); ``` -Step 7: Read documents +We can perform range queries as well. + +```sql +SELECT cursorPage FROM documentdb_api.find_cursor_first_page('documentdb', '{ "find" : "patient", "filter" : { "$and": [{ "age": { "$gte": 10 } },{ "age": { "$lte": 35 } }] }}'); +``` -```python +#### Update document in a collection -# Read all documents -for document in quickStartCollection.find(): - print(document) +DocumentDB uses the [documentdb_api.update](https://github.com/microsoft/documentdb/wiki/Functions#update) function to modify existing documents within a collection. -# Read a specific document -singleDocumentReadResult = quickStartCollection.find_one({'name': 'John Doe'}) - print(singleDocumentReadResult) +The SQL command updates the `age` for patient `P004`. +```sql +select documentdb_api.update('documentdb', '{"update":"patient", "updates":[{"q":{"patient_id":"P004"},"u":{"$set":{"age":14}}}]}'); ``` -Step 8: Run aggregation pipeline operation +Similarly, we can update multiple documents using `multi` property. -```python +```sql +SELECT documentdb_api.update('documentdb', '{"update":"patient", "updates":[{"q":{},"u":{"$set":{"age":24}},"multi":true}]}'); +``` + +#### Delete document from the collection + +DocumentDB uses the [documentdb_api.delete](https://github.com/microsoft/documentdb/wiki/Functions#delete) function for precise document removal based on specified criteria. + +The SQL command deletes the document for patient `P002`. -pipeline = [ - {'$match': {'name': 'Alice Johnson'}}, - {'$project': { - '_id': 0, - 'name': 1, - 'email': 1 - }} -] +```sql +SELECT documentdb_api.delete('documentdb', '{"delete": "patient", "deletes": [{"q": {"patient_id": "P002"}, "limit": 1}]}'); +``` + +### Collection management -results = quickStartCollection.aggregate(pipeline) -print("Aggregation results:") -for eachDocument in results: - print(eachDocument) +We can review for the available collections and databases by querying [documentdb_api.list_collections_cursor_first_page](https://github.com/microsoft/documentdb/wiki/Functions#list_collections_cursor_first_page). +```sql +SELECT * FROM documentdb_api.list_collections_cursor_first_page('documentdb', '{ "listCollections": 1 }'); ``` -### Helpful Links +[documentdb_api.list_indexes_cursor_first_page](https://github.com/microsoft/documentdb/wiki/Functions#list_indexes_cursor_first_page) allows reviewing for the existing indexes on a collection. We can find collection_id from `documentdb_api.list_collections_cursor_first_page`. + +```sql +SELECT documentdb_api.list_indexes_cursor_first_page('documentdb','{"listIndexes": "patient"}'); +``` -- Check out our [website](https://documentdb.io) to stay up to date with the latest on the project. -- Check out our [docs](https://documentdb.io/docs) for MongoDB API compatibility, quickstarts and more. -- Contributors and users can join the [DocumentDB Discord channel](https://discord.gg/vH7bYu524D) for quick collaboration. -- Check out [FerretDB](https://github.com/FerretDB/FerretDB) and their integration of DocumentDB as a backend engine. \ No newline at end of file +`ttl` indexes by default gets scheduled through the `pg_cron` scheduler, which could be reviewed by querying the `cron.job` table. + +```sql +select * from cron.job; +``` + +### Indexing + +#### Create an Index + +DocumentDB uses the `documentdb_api.create_indexes_background` function, which allows background index creation without disrupting database operations. + +The SQL command demonstrates how to create a `single field` index on `age` on the `patient` collection of the `documentdb`. + +```sql +SELECT * FROM documentdb_api.create_indexes_background('documentdb', '{ "createIndexes": "patient", "indexes": [{ "key": {"age": 1},"name": "idx_age"}]}'); +``` + +The SQL command demonstrates how to create a `compound index` on fields age and registration_year on the `patient` collection of the `documentdb`. + +```sql +SELECT * FROM documentdb_api.create_indexes_background('documentdb', '{ "createIndexes": "patient", "indexes": [{ "key": {"registration_year": 1, "age": 1},"name": "idx_regyr_age"}]}'); +``` + +#### Drop an Index + +DocumentDB uses the `documentdb_api.drop_indexes` function, which allows you to remove an existing index from a collection. The SQL command demonstrates how to drop the index named `id_ab_1` from the `first_collection` collection of the `documentdb`. + +```sql +CALL documentdb_api.drop_indexes('documentdb', '{"dropIndexes": "patient", "index":"idx_age"}'); +``` + +### Perform aggregations `Group by` + +DocumentDB provides the [documentdb_api.aggregate_cursor_first_page](https://github.com/microsoft/documentdb/wiki/Functions#aggregate_cursor_first_page) function, for performing aggregations over the document store. + +The example projects an aggregation on number of patients registered over the years. + +```sql +SELECT cursorpage FROM documentdb_api.aggregate_cursor_first_page('documentdb', '{ "aggregate": "patient", "pipeline": [ { "$group": { "_id": "$registration_year", "count_patients": { "$count": {} } } } ] , "cursor": { "batchSize": 3 } }'); +``` + +We can perform more complex operations, listing below a few more usage examples. +The example demonstrates an aggregation on patients, categorizing them into buckets defined by registration_year boundaries. + +```sql +SELECT cursorpage FROM documentdb_api.aggregate_cursor_first_page('documentdb', '{ "aggregate": "patient", "pipeline": [ { "$bucket": { "groupBy": "$registration_year", "boundaries": ["2022","2023","2024"], "default": "unknown" } } ], "cursor": { "batchSize": 3 } }'); +``` + +This query performs an aggregation on the `patient` collection to group documents by `registration_year`. It collects unique patient conditions for each registration year using the `$addToSet` operator. + +```sql +SELECT cursorpage FROM documentdb_api.aggregate_cursor_first_page('documentdb', '{ "aggregate": "patient", "pipeline": [ { "$group": { "_id": "$registration_year", "conditions": { "$addToSet": { "conditions" : "$conditions" } } } } ], "cursor": { "batchSize": 3 } }'); +``` + +### Join data from multiple collections + +Let's create an additional collection named `appointment` to demonstrate how a join operation can be performed. + +```sql +select documentdb_api.insert_one('documentdb','appointment', '{"appointment_id": "A001", "patient_id": "P001", "doctor_name": "Dr. Milind", "appointment_date": "2023-01-20", "reason": "Routine checkup" }'); +select documentdb_api.insert_one('documentdb','appointment', '{"appointment_id": "A002", "patient_id": "P001", "doctor_name": "Dr. Moore", "appointment_date": "2023-02-10", "reason": "Follow-up"}'); +select documentdb_api.insert_one('documentdb','appointment', '{"appointment_id": "A004", "patient_id": "P003", "doctor_name": "Dr. Smith", "appointment_date": "2024-03-12", "reason": "Allergy consultation"}'); +select documentdb_api.insert_one('documentdb','appointment', '{"appointment_id": "A005", "patient_id": "P004", "doctor_name": "Dr. Moore", "appointment_date": "2024-04-15", "reason": "Migraine treatment"}'); +select documentdb_api.insert_one('documentdb','appointment', '{"appointment_id": "A007","patient_id": "P001", "doctor_name": "Dr. Milind", "appointment_date": "2024-06-05", "reason": "Blood test"}'); +select documentdb_api.insert_one('documentdb','appointment', '{ "appointment_id": "A009", "patient_id": "P003", "doctor_name": "Dr. Smith","appointment_date": "2025-01-20", "reason": "Follow-up visit"}'); +``` + +The example presents each patient along with the doctors visited. + +```sql +SELECT cursorpage FROM documentdb_api.aggregate_cursor_first_page('documentdb', '{ "aggregate": "patient", "pipeline": [ { "$lookup": { "from": "appointment","localField": "patient_id", "foreignField": "patient_id", "as": "appointment" } },{"$unwind":"$appointment"},{"$project":{"_id":0,"name":1,"appointment.doctor_name":1,"appointment.appointment_date":1}} ], "cursor": { "batchSize": 3 } }'); +``` diff --git a/internal/pg_documentdb_distributed/src/test/regress/expected/documentdb_distributed_setup.out b/internal/pg_documentdb_distributed/src/test/regress/expected/documentdb_distributed_setup.out index 75d800fa8..b1ba8a1f7 100644 --- a/internal/pg_documentdb_distributed/src/test/regress/expected/documentdb_distributed_setup.out +++ b/internal/pg_documentdb_distributed/src/test/regress/expected/documentdb_distributed_setup.out @@ -3,10 +3,11 @@ SELECT datname, datcollate, datctype, pg_encoding_to_char(encoding), datlocprovi datname | datcollate | datctype | pg_encoding_to_char | datlocprovider --------------------------------------------------------------------- postgres | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c - regression | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c + ivorysql | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c template1 | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c template0 | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c -(4 rows) + regression | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c +(5 rows) CREATE OR REPLACE FUNCTION documentdb_distributed_test_helpers.latest_documentdb_distributed_version() RETURNS text diff --git a/packaging/build_packages.sh b/packaging/build_packages.sh index 6624cc069..53ecfc5d6 100755 --- a/packaging/build_packages.sh +++ b/packaging/build_packages.sh @@ -49,11 +49,11 @@ while [[ $# -gt 0 ]]; do ;; esac ;; - --pg) + --ivy) shift case $1 in - 15|16|17) - PG=$1 + 2|3|4) + PG=$(($1+13)) ;; *) echo "Invalid --pg value. Allowed values are [15, 16, 17]" diff --git a/pg_documentdb/src/test/regress/expected/documentdb_test_helpers.out b/pg_documentdb/src/test/regress/expected/documentdb_test_helpers.out index 644d9146f..2c8855f4f 100644 --- a/pg_documentdb/src/test/regress/expected/documentdb_test_helpers.out +++ b/pg_documentdb/src/test/regress/expected/documentdb_test_helpers.out @@ -1,11 +1,11 @@ CREATE SCHEMA documentdb_test_helpers; -SELECT datname, datcollate, datctype, pg_encoding_to_char(encoding), datlocprovider FROM pg_database; +SELECT datname, datcollate, datctype, pg_encoding_to_char(encoding), datlocprovider FROM pg_database where datname <> 'ivorysql' ORDER BY datname; datname | datcollate | datctype | pg_encoding_to_char | datlocprovider ------------+-------------+-------------+---------------------+---------------- postgres | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c regression | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c - template1 | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c template0 | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c + template1 | en_US.UTF-8 | en_US.UTF-8 | UTF8 | c (4 rows) -- Check if recreating the extension works diff --git a/pg_documentdb/src/test/regress/sql/documentdb_test_helpers.sql b/pg_documentdb/src/test/regress/sql/documentdb_test_helpers.sql index f60ec17b1..d40f883ff 100644 --- a/pg_documentdb/src/test/regress/sql/documentdb_test_helpers.sql +++ b/pg_documentdb/src/test/regress/sql/documentdb_test_helpers.sql @@ -1,6 +1,6 @@ CREATE SCHEMA documentdb_test_helpers; -SELECT datname, datcollate, datctype, pg_encoding_to_char(encoding), datlocprovider FROM pg_database; +SELECT datname, datcollate, datctype, pg_encoding_to_char(encoding), datlocprovider FROM pg_database where datname <> 'ivorysql' ORDER BY datname; -- Check if recreating the extension works DROP EXTENSION IF EXISTS documentdb; diff --git a/scripts/build_and_start_gateway.sh b/scripts/build_and_start_gateway.sh index 9b0609e28..17b845562 100755 --- a/scripts/build_and_start_gateway.sh +++ b/scripts/build_and_start_gateway.sh @@ -84,21 +84,21 @@ while [[ -L $source ]]; do done scriptDir="$(cd -P "$(dirname "$source")" && pwd)" -# Check if PostgreSQL is running with a timeout of 10 minutes +# Check if IvorySQL is running with a timeout of 10 minutes timeout=600 interval=5 elapsed=0 -echo "Waiting for PostgreSQL to be ready on $hostname:$port..." +echo "Waiting for IvorySQL to be ready on $hostname:$port..." while ! pg_isready -h "$hostname" -p "$port" > /dev/null 2>&1; do if [ "$elapsed" -ge "$timeout" ]; then - echo "PostgreSQL did not become ready within 10 minutes. Exiting." + echo "IvorySQL did not become ready within 10 minutes. Exiting." exit 1 fi sleep "$interval" elapsed=$((elapsed + interval)) done -echo "PostgreSQL is ready." +echo "IvorySQL is ready." if [ "$clean" = "true" ]; then echo "Cleaning the build directory..." diff --git a/scripts/install_setup_citus_core_oss.sh b/scripts/install_setup_citus_core_oss.sh index 305933933..436ca5cd8 100755 --- a/scripts/install_setup_citus_core_oss.sh +++ b/scripts/install_setup_citus_core_oss.sh @@ -22,8 +22,8 @@ scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" CITUS_REF=$(GetCitusVersion $citusVersion) . $scriptDir/utils.sh -if [ "${PGVERSION:-}" != "" ]; then - pgPath=$(GetPostgresPath $PGVERSION) +if [ "${IVORYVERSION:-}" != "" ]; then + pgPath=$(GetPostgresPath $IVORYVERSION) PATH=$pgPath:$PATH fi diff --git a/scripts/install_setup_pg_cron.sh b/scripts/install_setup_pg_cron.sh index 8ffb03dcb..b255f685d 100755 --- a/scripts/install_setup_pg_cron.sh +++ b/scripts/install_setup_pg_cron.sh @@ -20,7 +20,7 @@ scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" CITUS_PG_CRON_REF=$(GetPgCronVersion) . $scriptDir/utils.sh -pgBinDir=$(GetPostgresPath $PGVERSION) +pgBinDir=$(GetPostgresPath $IVORYVERSION) PATH=$pgBinDir:$PATH; pushd $INSTALL_DEPENDENCIES_ROOT diff --git a/scripts/install_setup_pgvector.sh b/scripts/install_setup_pgvector.sh index 629336a43..c8a02177f 100755 --- a/scripts/install_setup_pgvector.sh +++ b/scripts/install_setup_pgvector.sh @@ -18,7 +18,7 @@ scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" echo "scriptDir: $scriptDir" . $scriptDir/utils.sh -pgBinDir=$(GetPostgresPath $PGVERSION) +pgBinDir=$(GetPostgresPath $IVORYVERSION) PATH=$pgBinDir:$PATH; . $scriptDir/setup_versions.sh diff --git a/scripts/install_setup_postgis.sh b/scripts/install_setup_postgis.sh index 01af6ac5c..47b946422 100755 --- a/scripts/install_setup_postgis.sh +++ b/scripts/install_setup_postgis.sh @@ -27,7 +27,7 @@ scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" POSTGIS_REF=$(GetPostgisVersion) . $scriptDir/utils.sh -pgBinDir=$(GetPostgresPath $PGVERSION) +pgBinDir=$(GetPostgresPath $IVORYVERSION) POSTGIS_REPO=postgis-repo rm -rf $POSTGIS_REPO diff --git a/scripts/install_setup_postgres.sh b/scripts/install_setup_postgres.sh index 8dba78adb..dfced4608 100755 --- a/scripts/install_setup_postgres.sh +++ b/scripts/install_setup_postgres.sh @@ -5,14 +5,14 @@ set -e # fail if trying to reference a variable that is not set. set -u -postgresqlInstallDir="" +ivorysqlInstallDir="" debug="false" cassert="false" help="false"; -pgVersion="" +ivyVersion="" while getopts "d:hxcv:" opt; do case $opt in - d) postgresqlInstallDir="$OPTARG" + d) ivorysqlInstallDir="$OPTARG" ;; x) debug="true" ;; @@ -20,7 +20,7 @@ while getopts "d:hxcv:" opt; do ;; h) help="true" ;; - v) pgVersion="$OPTARG" + v) ivyVersion="$OPTARG" ;; esac @@ -34,18 +34,18 @@ while getopts "d:hxcv:" opt; do done if [ "$help" == "true" ]; then - echo "downloads postgresql-14.2 sources, build and install it." - echo "[-d] the directory to install postgresql to. Default: /usr/lib/postgresql/14" + echo "downloads IvorySQL-14.2 sources, build and install it." + echo "[-d] the directory to install IvorySQL to. Default: /usr/lib/IvorySQL/14" echo "[-x] build with debug symbols." exit 1; fi -if [ -z $postgresqlInstallDir ]; then +if [ -z $ivorysqlInstallDir ]; then echo "Postgres Install Directory must be specified." exit 1; fi -if [ -z $pgVersion ]; then +if [ -z $ivyVersion ]; then echo "PG Version must be specified"; exit 1; fi @@ -62,29 +62,29 @@ done scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptDir/setup_versions.sh -POSTGRESQL_REF=$(GetPostgresSourceRef $pgVersion) +IvorySQL_REF=$(GetPostgresSourceRef $ivyVersion) pushd $INSTALL_DEPENDENCIES_ROOT -rm -rf postgres-repo/$pgVersion -mkdir -p postgres-repo/$pgVersion -cd postgres-repo/$pgVersion +rm -rf postgres-repo/$ivyVersion +mkdir -p postgres-repo/$ivyVersion +cd postgres-repo/$ivyVersion git init -git remote add origin https://github.com/postgres/postgres +git remote add origin https://github.com/IvorySQL/IvorySQL # checkout to the commit specified in the cgmanifest.json -git fetch --depth 1 origin "$POSTGRESQL_REF" +git fetch --depth 1 origin "$IvorySQL_REF" git checkout FETCH_HEAD -echo "building and installing postgresql ref $POSTGRESQL_REF and installing to $postgresqlInstallDir..." +echo "building and installing IvorySQL ref $IvorySQL_REF and installing to $ivorysqlInstallDir..." if [ "$debug" == "true" ]; then - ./configure --enable-debug --enable-cassert --enable-tap-tests CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" --with-openssl --prefix="$postgresqlInstallDir" --with-icu + ./configure --enable-debug --enable-cassert --enable-tap-tests CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" --with-openssl --prefix="$ivorysqlInstallDir" --with-icu elif [ "$cassert" == "true" ]; then - ./configure --enable-debug --enable-cassert --enable-tap-tests --with-openssl --prefix="$postgresqlInstallDir" --with-icu + ./configure --enable-debug --enable-cassert --enable-tap-tests --with-openssl --prefix="$ivorysqlInstallDir" --with-icu else - ./configure --enable-debug --enable-tap-tests --with-openssl --prefix="$postgresqlInstallDir" --with-icu + ./configure --enable-debug --enable-tap-tests --with-openssl --prefix="$ivorysqlInstallDir" --with-icu fi make clean && make -sj$(cat /proc/cpuinfo | grep -c "processor") install diff --git a/scripts/install_setup_rum_oss.sh b/scripts/install_setup_rum_oss.sh index 535d0e05c..32a0b35f7 100755 --- a/scripts/install_setup_rum_oss.sh +++ b/scripts/install_setup_rum_oss.sh @@ -21,8 +21,8 @@ echo "scriptDir: $scriptDir" RUM_REF=$(GetRumVersion) . $scriptDir/utils.sh -if [ "${PGVERSION:-}" != "" ]; then - pgPath=$(GetPostgresPath $PGVERSION) +if [ "${IVORYVERSION:-}" != "" ]; then + pgPath=$(GetPostgresPath $IVORYVERSION) PATH=$pgPath:$PATH fi diff --git a/scripts/install_setup_system_rows.sh b/scripts/install_setup_system_rows.sh index 3282401c5..f67e63bf1 100755 --- a/scripts/install_setup_system_rows.sh +++ b/scripts/install_setup_system_rows.sh @@ -17,7 +17,7 @@ done scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptDir/utils.sh . $scriptDir/setup_versions.sh -POSTGRESQL_REF=$(GetPostgresSourceRef $PGVERSION) +POSTGRESQL_REF=$(GetPostgresSourceRef $IVORYVERSION) pushd $INSTALL_DEPENDENCIES_ROOT @@ -26,13 +26,13 @@ mkdir postgres-repo-for-system-rows cd postgres-repo-for-system-rows git init -git remote add origin https://github.com/postgres/postgres +git remote add origin https://github.com/IvorySQL/IvorySQL # checkout to the commit specified in the cgmanifest.json git fetch --depth 1 origin "$POSTGRESQL_REF" git checkout FETCH_HEAD -pgBinDir=$(GetPostgresPath $PGVERSION) +pgBinDir=$(GetPostgresPath $IVORYVERSION) PATH=$pgBinDir:$PATH; echo "building and installing tsm_system_rows extension with pg path $pgBinDir ..." diff --git a/scripts/setup_versions.sh b/scripts/setup_versions.sh index feb0d5939..477cce5d3 100755 --- a/scripts/setup_versions.sh +++ b/scripts/setup_versions.sh @@ -8,11 +8,11 @@ set -e # declare all the versions of dependencies LIBBSON_VERSION=1.28.0 # This maps to REL_17_4:f8554dee417ffc4540c94cf357f7bf7d4b6e5d80 -POSTGRES_17_REF="REL_17_4" +IvorySQL_4_REF="IvorySQL_4.5" # This maps to REL_16_8:71eb35c0b18de96537bd3876ec9bf8075bfd484f -POSTGRES_16_REF="REL_16_8" +IvorySQL_3_REF="IvorySQL_3.4" # This maps to REL15_12:50d3d22baba63613d1f1406b2ed460dc9b03c3fc -POSTGRES_15_REF="REL_15_12" +IvorySQL_2_REF="Ivory_REL_2_3" # This is commit f3cb3d99eed7dd3db82104ab2a4a13b1233d0023 CITUS_VERSION=v12.1.7 # This is commit d28a5eae6c78935313824d319480632783d48d10 @@ -31,15 +31,15 @@ UNCRUSTIFY_VERSION=uncrustify-0.68.1 function GetPostgresSourceRef() { - local pgVersion=$1 - if [ "$pgVersion" == "17" ]; then - echo $POSTGRES_17_REF - elif [ "$pgVersion" == "16" ]; then - echo $POSTGRES_16_REF - elif [ "$pgVersion" == "15" ]; then - echo $POSTGRES_15_REF + local ivyVersion=$1 + if [ "$ivyVersion" == "4" ]; then + echo $IvorySQL_4_REF + elif [ "$ivyVersion" == "3" ]; then + echo $IvorySQL_3_REF + elif [ "$ivyVersion" == "2" ]; then + echo $IvorySQL_2_REF else - echo "Invalid PG Version specified $pgVersion"; + echo "Invalid PG Version specified $ivyVersion"; exit 1; fi } @@ -47,7 +47,7 @@ function GetPostgresSourceRef() function GetCitusVersion() { local citusVersion=$1 - if [ "$PGVERSION" == "17" ]; then + if [ "$IVORYVERSION" == "4" ]; then echo $CITUS_13_VERSION elif [ "$citusVersion" == "13" ] || [ "$citusVersion" == "v13.0" ] || [ "$citusVersion" == "$CITUS_13_VERSION" ]; then echo $CITUS_13_VERSION diff --git a/scripts/start_oss_server.sh b/scripts/start_oss_server.sh index c56d5f40e..fd6d0a44f 100755 --- a/scripts/start_oss_server.sh +++ b/scripts/start_oss_server.sh @@ -5,9 +5,9 @@ set -e # fail if trying to reference a variable that is not set. set -u -PG_VERSION=${PG_VERSION_USED:-16} +IVY_VERSION=${IVY_VERSION_USED:-4} coordinatorPort="9712" -postgresDirectory="" +ivoryDirectory="" initSetup="false" forceCleanup="false" help="false" @@ -16,7 +16,7 @@ distributed="false" allowExternalAccess="false" while getopts "d:p:hcsxe" opt; do case $opt in - d) postgresDirectory="$OPTARG" + d) ivoryDirectory="$OPTARG" ;; c) initSetup="true" forceCleanup="true" @@ -53,7 +53,7 @@ if [ "$help" == "true" ]; then echo "${green}[-c] - optional argument. FORCE cleanup - removes all existing data and reinitializes" echo "${green}[-s] - optional argument. Stops all servers and exits" echo "${green}[-x] - start oss server with documentdb_distributed extension" - echo "${green}[-e] - optional argument. Allows PostgreSQL access from any IP address" + echo "${green}[-e] - optional argument. Allows IvorySQL access from any IP address" echo "${green}[-p ] - optional argument. specifies the port for the coordinator" echo "${green}if postgresDir not specified assumed to be /data" exit 1; @@ -96,56 +96,56 @@ scriptDir="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptDir/utils.sh -if [ -z $postgresDirectory ]; then - postgresDirectory="/data" +if [ -z $ivoryDirectory ]; then + ivoryDirectory="/data" fi -# Only initialize if directory doesn't exist, is empty, or doesn't contain a valid PostgreSQL data directory -# Check for PG_VERSION file which indicates a valid PostgreSQL data directory -if [ ! -d "$postgresDirectory" ]; then +# Only initialize if directory doesn't exist, is empty, or doesn't contain a valid IvorySQL data directory +# Check for IVY_VERSION file which indicates a valid IvorySQL data directory +if [ ! -d "$ivoryDirectory" ]; then # Directory doesn't exist, we need to initialize - echo "${green}Directory $postgresDirectory doesn't exist, will initialize PostgreSQL data directory${reset}" + echo "${green}Directory $ivoryDirectory doesn't exist, will initialize IvorySQL data directory${reset}" initSetup="true" -elif [ ! -f "$postgresDirectory/PG_VERSION" ]; then - # Directory exists but no PG_VERSION file - if [ "$(ls -A "$postgresDirectory" 2>/dev/null)" ]; then - # Directory exists and is not empty but doesn't have PG_VERSION +elif [ ! -f "$ivoryDirectory/IVY_VERSION" ]; then + # Directory exists but no IVY_VERSION file + if [ "$(ls -A "$ivoryDirectory" 2>/dev/null)" ]; then + # Directory exists and is not empty but doesn't have IVY_VERSION # This might be a corrupted or incompatible data directory - echo "${red}Warning: Directory $postgresDirectory exists but doesn't appear to contain a valid PostgreSQL data directory.${reset}" + echo "${red}Warning: Directory $ivoryDirectory exists but doesn't appear to contain a valid IvorySQL data directory.${reset}" echo "${red}Use -c flag to force cleanup and re-initialization, or specify a different directory with -d.${reset}" exit 1 else # Directory exists but is empty, we can initialize - echo "${green}Directory $postgresDirectory is empty, will initialize PostgreSQL data directory${reset}" + echo "${green}Directory $ivoryDirectory is empty, will initialize IvorySQL data directory${reset}" initSetup="true" fi else - # Directory exists and has PG_VERSION, check if it's compatible - echo "${green}Found existing PostgreSQL data directory at $postgresDirectory${reset}" + # Directory exists and has IVY_VERSION, check if it's compatible + echo "${green}Found existing IvorySQL data directory at $ivoryDirectory${reset}" fi # We stop the coordinator first and the worker node servers # afterwards. However this order is not required and it doesn't # really matter which order we choose to stop the active servers. echo "${green}Stopping any existing postgres servers${reset}" -StopServer $postgresDirectory +StopServer $ivoryDirectory if [ "$stop" == "true" ]; then exit 0; fi -echo "InitDatabaseExtended $initSetup $postgresDirectory" +echo "InitDatabaseExtended $initSetup $ivoryDirectory" if [ "$initSetup" == "true" ]; then - InitDatabaseExtended $postgresDirectory "$preloadLibraries" + InitDatabaseExtended $ivoryDirectory "$preloadLibraries" fi -# Update PostgreSQL configuration to allow access from any IP +# Update IvorySQL configuration to allow access from any IP if [ "$allowExternalAccess" == "true" ]; then - postgresConfigFile="$postgresDirectory/postgresql.conf" - hbaConfigFile="$postgresDirectory/pg_hba.conf" + postgresConfigFile="$ivoryDirectory/postgresql.conf" + hbaConfigFile="$ivoryDirectory/pg_hba.conf" - echo "${green}Configuring PostgreSQL to allow access from any IP address${reset}" + echo "${green}Configuring IvorySQL to allow access from any IP address${reset}" echo "listen_addresses = '*'" >> $postgresConfigFile echo "host all all 0.0.0.0/0 scram-sha-256" >> $hbaConfigFile echo "host all all ::0/0 scram-sha-256" >> $hbaConfigFile @@ -155,7 +155,7 @@ userName=$(whoami) sudo mkdir -p /var/run/postgresql sudo chown -R $userName:$userName /var/run/postgresql -StartServer $postgresDirectory $coordinatorPort +StartServer $ivoryDirectory $coordinatorPort if [ "$initSetup" == "true" ]; then SetupPostgresServerExtensions "$userName" $coordinatorPort $extensionName diff --git a/scripts/utils.sh b/scripts/utils.sh index abd7bf580..3e4fc4fa0 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -21,43 +21,37 @@ trap 'error ${LINENO}' ERR function GetPostgresPath() { - local pgVersion=$1 - local osVersion=$(cat /etc/os-release | grep "^ID="); - - if [[ "$osVersion" == "ID=ubuntu" || "$osVersion" == "ID=debian"|| "$osVersion" == "ID=mariner" || "$osVersion" == "ID=azurelinux" ]]; then - echo "/usr/lib/postgresql/$pgVersion/bin" - else - echo "/usr/pgsql-$pgVersion/bin" - fi + local ivyVersion=$1 + echo "/var/local/ivorysql/ivory-$ivyVersion/bin" } function GetPostgresSourceRef() { - local pgVersion=$1 - if [ "$pgVersion" == "16" ]; then + local ivyVersion=$1 + if [ "$ivyVersion" == "3" ]; then # This maps to REL_16_2:b78fa8547d02fc72ace679fb4d5289dccdbfc781 - POSTGRESQL_REF="REL_16_2" - elif [ "$pgVersion" == "15" ]; then + IvorySQL_REF="IvorySQL_3.4" + elif [ "$ivyVersion" == "2" ]; then # This maps to REL15_3:8382864eb5c9f9ebe962ac20b3392be5ae304d23 - POSTGRESQL_REF="REL_15_3" + IvorySQL_REF="Ivory_REL_2_3" else - echo "Invalid PG Version specified $pgVersion"; + echo "Invalid PG Version specified $ivyVersion"; exit 1; fi - echo $POSTGRESQL_REF + echo $IvorySQL_REF } function GetPGCTL() { - local pgVersion=${PG_VERSION:-16} - echo ${pgctlPath:-$(GetPostgresPath $pgVersion)/pg_ctl} + local ivyVersion=${IVY_VERSION:-4} + echo ${pgctlPath:-$(GetPostgresPath $ivyVersion)/pg_ctl} } function GetInitDB() { - local pgVersion=${PG_VERSION:-16} - echo $(GetPostgresPath $pgVersion)/initdb + local ivyVersion=${IVY_VERSION:-4} + echo $(GetPostgresPath $ivyVersion)/initdb } function StopServer() @@ -106,7 +100,7 @@ function InitDatabaseExtended() local _directory=$1 local _preloadLibraries=$2 - echo "Initializing PostgreSQL database in $_directory with preload libraries: $_preloadLibraries" + echo "Initializing IvorySQL database in $_directory with preload libraries: $_preloadLibraries" if [ -d "$_directory" ]; then echo "Removing contents of $_directory"