Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
ARG POSTGRES_INSTALL_ARG=
ARG PG_VERSION=16
ARG IVORY_VERSION=4
ARG CITUS_VERSION=12

# declare installed PG version and Citus version
ENV PG_VERSION=${PG_VERSION}
ENV IVORY_VERSION=${IVORY_VERSION}
ENV CITUS_VERSION=${CITUS_VERSION}

# Install build essentials - Compiler, debugger, make, etc.
Expand Down Expand Up @@ -35,6 +35,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
libtool \
libicu-dev \
libssl-dev \
libxml2-dev \
&& rm -rf /var/lib/apt/lists/*

# Add pgdg repo
Expand All @@ -60,27 +61,27 @@ COPY scripts/utils.sh /tmp/install_setup/

# Install postgres
COPY scripts/install_setup_postgres.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/usr/lib/postgresql/${PG_VERSION}\" $POSTGRES_INSTALL_ARG -v ${PG_VERSION}" ]
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/var/local/ivorysql/ivory-${IVORY_VERSION}\" $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION}" ]

# Install RUM from source
COPY scripts/install_setup_rum_oss.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ]

# Install citus
COPY scripts/install_setup_citus_core_oss.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ]

# Install citus-indent
COPY scripts/install_citus_indent.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_citus_indent.sh" ]

# Install SYSTEM_ROWS
COPY scripts/install_setup_system_rows.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh" ]

# Install PG_CRON
COPY scripts/install_setup_pg_cron.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ]

# Download Decimal128 Intel library
COPY scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/
Expand All @@ -92,7 +93,7 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_pcre2.sh" ]

# Install PG_VECTOR
COPY scripts/install_setup_pgvector.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh" ]

# Install PostGIS from source
RUN apt-get update && \
Expand All @@ -105,7 +106,7 @@ RUN apt-get update && \
&& rm -rf /var/lib/apt/lists/*

COPY scripts/install_setup_postgis.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh" ]

# locale
RUN rm -rf /var/lib/apt/lists/* \
Expand All @@ -121,10 +122,10 @@ RUN echo "%sudo ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/no-pass-ask

# Add postgres to path for sudo commands.
# Add path for sudoers
RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/usr\/lib\/postgresql\/$PG_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path
RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/var\/local\/ivorysql\/ivory-$IVORY_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path

# Add PG to the path
ENV PATH=$PATH:/usr/lib/postgresql/$PG_VERSION/bin
ENV PATH=$PATH:/var/local/ivorysql/ivory-$IVORY_VERSION/bin

USER documentdb
WORKDIR /home/documentdb
36 changes: 17 additions & 19 deletions .github/containers/Build-Ubuntu/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
ARG POSTGRES_INSTALL_ARG=
ARG PG_VERSION=16
ARG IVORY_VERSION=4
ARG CITUS_VERSION=12

# declare installed PG version and Citus version
ENV PG_VERSION=${PG_VERSION}
ENV IVORY_VERSION=${IVORY_VERSION}
ENV CITUS_VERSION=${CITUS_VERSION}

# Install build essentials - Compiler, debugger, make, etc.
Expand All @@ -20,8 +20,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
pkg-config \
cmake \
git \
locales \
gcc \
locales \
gcc \
gdb \
libipc-run-perl \
unzip \
Expand All @@ -35,6 +35,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
libtool \
libicu-dev \
libssl-dev \
libxml2-dev \
&& rm -rf /var/lib/apt/lists/*

# Add pgdg repo
Expand All @@ -60,27 +61,27 @@ COPY scripts/utils.sh /tmp/install_setup/

# Install postgres
COPY scripts/install_setup_postgres.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/usr/lib/postgresql/${PG_VERSION}\" $POSTGRES_INSTALL_ARG -v ${PG_VERSION}" ]
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_postgres.sh -d \"/var/local/ivorysql/ivory-${IVORY_VERSION}\" $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION}" ]

# Install RUM from source
COPY scripts/install_setup_rum_oss.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh" ]

# Install citus
COPY scripts/install_setup_citus_core_oss.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}" ]

# Install citus-indent
COPY scripts/install_citus_indent.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "/tmp/install_setup/install_citus_indent.sh" ]

# Install SYSTEM_ROWS
COPY scripts/install_setup_system_rows.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh" ]

# Install PG_CRON
COPY scripts/install_setup_pg_cron.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh" ]

# Download Decimal128 Intel library
COPY scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/
Expand All @@ -92,19 +93,20 @@ RUN [ "bin/bash", "-c", "/tmp/install_setup/install_setup_pcre2.sh" ]

# Install PG_VECTOR
COPY scripts/install_setup_pgvector.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh" ]

# Install PostGIS from source
RUN add-apt-repository -y ppa:ubuntugis/ppa && apt-get update && \
RUN apt-get update && \
apt-get install -qy \
libproj-dev \
libxml2-dev \
libjson-c-dev \
libgeos-dev \
libgeos++-dev \
libgeos-dev \
&& rm -rf /var/lib/apt/lists/*

COPY scripts/install_setup_postgis.sh /tmp/install_setup/
RUN [ "bin/bash", "-c", "PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh" ]
RUN [ "bin/bash", "-c", "IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh" ]

# locale
RUN rm -rf /var/lib/apt/lists/* \
Expand All @@ -120,14 +122,10 @@ RUN echo "%sudo ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/no-pass-ask

# Add postgres to path for sudo commands.
# Add path for sudoers
RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/usr\/lib\/postgresql\/$PG_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path
RUN cat /etc/sudoers | grep secure_path | sed "s/\:\/bin\:/\:\/bin\:\/var\/local\/ivorysql\/ivory-$IVORY_VERSION\/bin\:/" >> /etc/sudoers.d/postgres_path

# Add PG to the path
ENV PATH=$PATH:/usr/lib/postgresql/$PG_VERSION/bin
ENV PATH=$PATH:/var/local/ivorysql/ivory-$IVORY_VERSION/bin

USER documentdb
WORKDIR /home/documentdb

LABEL org.opencontainers.image.source=https://github.com/microsoft/documentdb
LABEL org.opencontainers.image.description="DocumentDB ubuntu build image"
LABEL org.opencontainers.image.licenses=MIT
17 changes: 8 additions & 9 deletions .github/workflows/build_packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,8 @@ jobs:
runner: ubuntu-24.04
- arch: arm64
runner: ubuntu-24.04-arm
pg_version:
- 16
- 17
ivory_version:
- 4

steps:
- name: Checkout repository
Expand All @@ -56,12 +55,12 @@ jobs:

- name: Build Debian Package
run: |
./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg_version }} --version ${{ env.DOCUMENTDB_VERSION }}
./packaging/build_packages.sh --os ${{ matrix.os }} --ivy ${{ matrix.ivory_version }} --version ${{ env.DOCUMENTDB_VERSION }}

- name: Upload Debian Package as Artifact
uses: actions/upload-artifact@v4
with:
name: archive-${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }}
name: archive-${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.ivory_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }}
path: |
packaging/*.deb
retention-days: 7
Expand Down Expand Up @@ -121,10 +120,10 @@ jobs:
push: true
platforms: linux/${{ matrix.arch }}
file: .github/containers/Build-Ubuntu/Dockerfile_prebuild
tags: ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }}
tags: ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }}
build-args: |
BASE_IMAGE=${{ env.BASE_IMAGE }}
POSTGRES_VERSION=${{ matrix.pg_version }}
IVORY_VERSION=${{ matrix.ivory_version }}
DEB_PACKAGE_REL_PATH=${{ env.PACKAGE_NAME }}
labels: |
org.opencontainers.image.source=https://github.com/microsoft/documentdb
Expand All @@ -138,7 +137,7 @@ jobs:
- name: Sign manifest (keyless)
if: startsWith(github.ref, 'refs/tags/')
run: |
cosign sign ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} -y
cosign sign ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }} -y

- name: Verify manifest signature (keyless)
if: startsWith(github.ref, 'refs/tags/')
Expand All @@ -147,4 +146,4 @@ jobs:
cosign verify \
--certificate-identity-regexp "https://github.com/${{ github.repository }}/.github/workflows/build_packages.yml@refs/(heads|tags)/${{ github.ref_name }}" \
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.pg_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }}
ghcr.io/${{ github.repository }}/documentdb-oss:PG${{ matrix.ivory_version }}-${{ matrix.arch }}-${{ env.DOCUMENTDB_VERSION }}
9 changes: 4 additions & 5 deletions .github/workflows/build_rpm_packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,8 @@ jobs:
include:
- arch: amd64
runner: ubuntu-24.04
pg_version:
- 16
- 17
ivory_version:
- 4

steps:
- name: Checkout repository
Expand All @@ -50,12 +49,12 @@ jobs:

- name: Build RPM Package
run: |
./packaging/build_packages.sh --os ${{ matrix.os }} --pg ${{ matrix.pg_version }} --version ${{ env.DOCUMENTDB_VERSION }} --test-clean-install
./packaging/build_packages.sh --os ${{ matrix.os }} --ivy ${{ matrix.ivory_version }} --version ${{ env.DOCUMENTDB_VERSION }} --test-clean-install

- name: Upload RPM Package as Artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }}
name: ${{ matrix.os }}-${{ matrix.arch }}-pg${{ matrix.ivory_version }}-documentdb-${{ env.DOCUMENTDB_VERSION }}
path: |
packaging/*.rpm
retention-days: 7
Expand Down
34 changes: 16 additions & 18 deletions .github/workflows/regress_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,8 @@ jobs:
strategy:
fail-fast: false
matrix:
pg_version:
- 15
- 16
- 17
ivory_version:
- 4
arch:
- amd64
- arm64
Expand All @@ -51,7 +49,7 @@ jobs:

- name: Install package dependencies
run: |
export PG_VERSION=${{ matrix.pg_version }}
export IVORY_VERSION=${{ matrix.ivory_version }}
export CITUS_VERSION=12

export LC_ALL=en_US.UTF-8
Expand Down Expand Up @@ -100,23 +98,23 @@ jobs:
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT MAKE_PROGRAM=cmake /tmp/install_setup/install_setup_libbson.sh
cp ./scripts/utils.sh /tmp/install_setup
cp ./scripts/install_setup_postgres.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_postgres.sh -d /usr/lib/postgresql/${PG_VERSION} $POSTGRES_INSTALL_ARG -v ${PG_VERSION}
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_postgres.sh -d /var/local/ivorysql/ivory-${IVORY_VERSION} $POSTGRES_INSTALL_ARG -v ${IVORY_VERSION}
cp ./scripts/install_setup_rum_oss.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_rum_oss.sh
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_rum_oss.sh
cp ./scripts/install_setup_citus_core_oss.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_citus_core_oss.sh ${CITUS_VERSION}
cp ./scripts/install_citus_indent.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_citus_indent.sh
cp ./scripts/install_setup_system_rows.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_system_rows.sh
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_system_rows.sh
cp ./scripts/install_setup_pg_cron.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pg_cron.sh
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pg_cron.sh
cp ./scripts/install_setup_intel_decimal_math_lib.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_intel_decimal_math_lib.sh
cp ./scripts/install_setup_pcre2.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT /tmp/install_setup/install_setup_pcre2.sh
cp ./scripts/install_setup_pgvector.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_pgvector.sh
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_pgvector.sh

sudo apt-get update
sudo apt-get install -qy \
Expand All @@ -128,17 +126,17 @@ jobs:
libgeos-dev \

cp ./scripts/install_setup_postgis.sh /tmp/install_setup/
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT PGVERSION=$PG_VERSION /tmp/install_setup/install_setup_postgis.sh
sudo INSTALL_DEPENDENCIES_ROOT=$INSTALL_DEPENDENCIES_ROOT IVORYVERSION=$IVORY_VERSION /tmp/install_setup/install_setup_postgis.sh

- name: Export pg_config PATH
run: |
echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH
echo "/usr/lib/postgresql/${{ matrix.ivory_version }}/bin" >> $GITHUB_PATH

- name: Configure, Build and Install DocumentDB
run: |
which pg_config
make
sudo PATH=$PATH make install
env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make
sudo env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make install

- name: Run Tests
run: |
Expand All @@ -147,11 +145,11 @@ jobs:
export LANGUAGE=en_US
export LC_COLLATE=en_US.UTF-8
export LC_CTYPE=en_US.UTF-8
make check
env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" make check

- name: Citus Indent
run: |
citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1)
env "PATH=/var/local/ivorysql/ivory-4/bin:$PATH" citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1)

- name: Validate sanity of files
run: |
Expand All @@ -161,6 +159,6 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: logs_$${{ matrix.runner }}_PG${{ matrix.pg_version }}
name: logs_$${{ matrix.runner }}_PG${{ matrix.ivory_version }}
overwrite: true
path: "**/*.log"
Loading