diff --git a/examples/ror-with-kibana-reverse-proxy-demo/docker-compose.yml b/examples/ror-with-kibana-reverse-proxy-demo/docker-compose.yml index f949ba1..3515ac4 100644 --- a/examples/ror-with-kibana-reverse-proxy-demo/docker-compose.yml +++ b/examples/ror-with-kibana-reverse-proxy-demo/docker-compose.yml @@ -124,6 +124,7 @@ services: ELASTICSEARCH_ADDRESS: https://es-ror:9200 ELASTICSEARCH_USER: kibana ELASTICSEARCH_PASSWORD: kibana + KIBANA_ADDRESS: https://kbn-ror:5601 healthcheck: test: "test -f /tmp/init_done || exit 1" interval: 10s diff --git a/ror-demo-cluster/.env-showcase b/ror-demo-cluster/.env-showcase index 345c8e7..47a9978 100644 --- a/ror-demo-cluster/.env-showcase +++ b/ror-demo-cluster/.env-showcase @@ -5,10 +5,12 @@ # Dockerfile-use-ror-binaries-from-api - download ROR plugin from API (requires ROR_ES_VERSION / ROR_KBN_VERSION) # Dockerfile-use-ror-binaries-from-file - use a local plugin file (requires ES_ROR_FILE / KBN_ROR_FILE) -#ES_VERSION=8.19.11 -#ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-file -#ES_ROR_FILE=readonlyrest-1.69.0-pre01_es8.19.11.zip +ES_VERSION=8.19.11 +#ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-api +#ROR_ES_VERSION=1.69.1 +ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-file +ES_ROR_FILE=readonlyrest-1.70.0-pre10_es8.19.11.zip -#KBN_VERSION=8.19.11 -#KBN_DOCKERFILE=Dockerfile-use-ror-binaries-from-api -#ROR_KBN_VERSION=1.68.0 +KBN_VERSION=8.19.11 +KBN_DOCKERFILE=Dockerfile-use-ror-binaries-from-file +KBN_ROR_FILE=readonlyrest_kbn_universal-1.70.0-pre13_es8.19.11.zip diff --git a/ror-demo-cluster/conf/es/readonlyrest.yml b/ror-demo-cluster/conf/es/readonlyrest.yml index 849dd87..1da5b4b 100644 --- a/ror-demo-cluster/conf/es/readonlyrest.yml +++ b/ror-demo-cluster/conf/es/readonlyrest.yml @@ -11,66 +11,48 @@ readonlyrest: auth_key: kibana:kibana verbosity: error - - name: "Admins" - groups: [Administrators] + - name: "admin" + type: allow + users: ["admin"] + groups: ["*"] kibana: - access: admin + access: unrestricted + index: .kibana_@{acl:current_group} + hide_apps: ["app1"] + + - name: "Group1 users" + type: allow + groups_or: ["g1"] + kibana: + access: rw + index: .kibana_@{acl:current_group} + hide_apps: ["app2"] - - name: "End users" - groups: ["EndUsers"] - indices: ["*-frontend-*", "kibana_sample_data_*"] + - name: "Group2 users" + type: allow + groups_or: ["g2"] kibana: - index: .kibana_end_@{user} access: rw - hide_apps: ["Security", "Observability"] + index: .kibana_@{acl:current_group} + hide_apps: ["app3"] - - name: "Business users" - groups: ["BusinessUsers"] - indices: ["*-business-*", "kibana_sample_data_*"] + - name: "NO tenancy user" + type: allow + auth_key: user3:test kibana: - index: .kibana_business_@{user} - access: ro - hide_apps: ["Security", "Observability"] + access: rw + # default kibana index + hide_apps: ["app4"] users: - username: admin auth_key: admin:admin - groups: - - id: "Administrators" - name: "Administrators" - - id: "EndUsers" - name: "End Users" - - id: "BusinessUsers" - name: "Business Users" + groups: [admin, g1, g2] - username: user1 auth_key: user1:test - groups: - - id: "EndUsers" - name: "End Users" - - id: "BusinessUsers" - name: "Business Users" + groups: [g1, g2] - username: user2 auth_key: user2:test - groups: - - id: "EndUsers" - name: "End Users" - - - username: "*" - ror_kbn_auth: - name: "kbn1" - groups: ["*"] - groups: - - local_group: - id: "EndUsers" - name: "End Users" - external_group_ids: [ "extEndUsers" ] - - local_group: - id: "BusinessUsers" - name: "Business Users" - external_group_ids: [ "extBusinessUsers" ] - - ror_kbn: - - name: kbn1 - signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf" + groups: [g1] \ No newline at end of file diff --git a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml index 5f396db..b5262ee 100644 --- a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml +++ b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml @@ -14,53 +14,5 @@ server.ssl.redirectHttpFromPort: 80 xpack.encryptedSavedObjects.encryptionKey: "min-32-byte-long-strong-encryption-key" -readonlyrest_kbn.logLevel: info +readonlyrest_kbn.logLevel: trace readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' -readonlyrest_kbn: - auth: - signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf" - saml_keycloak: - buttonName: 'Keycloak SAML' - enabled: true - type: 'saml' - issuer: 'ror-saml' - entryPoint: 'http://kc.localhost:8080/realms/ror/protocol/saml' - kibanaExternalHost: 'localhost:15601' - protocol: 'https' - usernameParameter: 'nameID' - groupsParameter: 'Role' - logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/saml' - YOU_SHOULD_READ_ME_IN_STRATEGY_CONFIGURATION_LOG: 'unknown conf params should be passed unmodified to the underlying passport-saml library' - cert: 'MIICrDCCAZQCCQDN5Tcc+Rn6rTANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1rZXljbG9hay1zYW1sMB4XDTI1MTExMDA0MjQyOFoXDTM1MTEwODA0MjQyOFowGDEWMBQGA1UEAwwNa2V5Y2xvYWstc2FtbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv0EFcTjNkpn5kV/XedCJ/AQKsPafZ7u33u3zfYgZbTh0V+CJ4bUAZyTfvGrcxR/iZy3hkYQBL7DGM6c/fYJGq1IH+/jxAk/GBY9tVnxotRsyhIMJYtZCb+DBUVX+wLowv2kZPlf/6OibjVpy+I6klQxIU8aeMkd+M/Phl97l+pRUjTuQZvEaEtkVLcsG6gEUcNo2X0MpNFwT7UgpGZLGy8zSHGNu703tb8w0oCYTEj42WqLWYNm5NMqD/clbPRj8g+1qZHpSoIH+p/cqVfU+oAZd3R2Y8SGR2OhYwyu9wHRnuqgiEgCIYbzwyh3IzX+57R8MQxHnjBFnzJKFtBWHMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAUGrj7hEEHJvm58RYTjqhcJ38MVkgEmAnGovObxvS9Xi6ZBct4irWdpyESowRJtY4nxFGs0uiArOpQDkteSeDzJs/IkSg3xTx3UwOpevf8IV5gU8Bwq68Fyh9An8NvczE5XhLZD1Tacphlz5OzoXthMsS1pcCumr5ZELwMtvLYAWkEV/cKmlis4JuMEZXc5v9KVybsmHv1hgM+fxg1neuWrPK1JebuVm4oaHUHYKCxqn9MXjnDvOq6MkYtKBfcYf6BKk29lapYuHNiRTi0hMPD1tWVaZg3H1/uMxlLXDxAZqkS4DNq/7MeIMUWemOqxQHZAi4rtplrVl/F3WES6pqWw==' - oidc_keycloak: - buttonName: "Keycloak OIDC" - type: "oidc" - protocol: "https" - issuer: 'http://kc.localhost:8080/realms/ror' - authorizationURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/auth' - tokenURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/token' - userInfoURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/userinfo' - jwksURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/certs' - clientID: 'ror-oidc' - clientSecret: 'kibanasecret123' - scope: 'openid profile email' - usernameParameter: 'preferred_username' - groupsParameter: 'groups' - kibanaExternalHost: 'localhost:15601' - logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/logout' - oidc_lemon_ldap: - buttonName: "LemonLDAP OpenID" - type: "oidc" - protocol: "https" - issuer: 'https://oidctest.wsweet.org/' - authorizationURL: 'https://oidctest.wsweet.org/oauth2/authorize' - tokenURL: 'https://oidctest.wsweet.org/oauth2/token' - userInfoURL: 'https://oidctest.wsweet.org/oauth2/userinfo' - clientID: 'private' - clientSecret: 'tardis' - scope: 'openid users roles' - usernameParameter: 'sub' - groupsParameter: 'roles' - kibanaExternalHost: 'localhost:15601' - logoutUrl: 'https://oidctest.wsweet.org/oauth2/logout' - jwksURL: 'https://oidctest.wsweet.org/oauth2/jwks' diff --git a/ror-demo-cluster/docker-compose.yml b/ror-demo-cluster/docker-compose.yml index 1675ade..83b95f0 100644 --- a/ror-demo-cluster/docker-compose.yml +++ b/ror-demo-cluster/docker-compose.yml @@ -40,7 +40,7 @@ services: - node.name=es-ror-single - discovery.type=single-node - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=1 -Dcom.readonlyrest.settings.loading.delay=0s" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=1 -Dcom.readonlyrest.settings.loading.delay=1" - ES_VERSION=${ES_VERSION:-ES_VERSION_NOT_CONFIGURED} healthcheck: test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 2 --retry-connrefused -u admin:admin https://127.0.0.1:9200/_cluster/health >/dev/null || exit 1"] @@ -101,6 +101,7 @@ services: ELASTICSEARCH_ADDRESS: https://es-ror:9200 ELASTICSEARCH_USER: kibana ELASTICSEARCH_PASSWORD: kibana + KIBANA_ADDRESS: https://kbn-ror:5601 healthcheck: test: "test -f /tmp/init_done || exit 1" interval: 10s diff --git a/ror-demo-cluster/readonlyrest-1.70.0-pre10_es8.19.11.zip b/ror-demo-cluster/readonlyrest-1.70.0-pre10_es8.19.11.zip new file mode 100644 index 0000000..08fe55b Binary files /dev/null and b/ror-demo-cluster/readonlyrest-1.70.0-pre10_es8.19.11.zip differ diff --git a/ror-demo-cluster/readonlyrest_kbn_universal-1.70.0-pre13_es8.19.11.zip b/ror-demo-cluster/readonlyrest_kbn_universal-1.70.0-pre13_es8.19.11.zip new file mode 100644 index 0000000..29baac7 Binary files /dev/null and b/ror-demo-cluster/readonlyrest_kbn_universal-1.70.0-pre13_es8.19.11.zip differ diff --git a/shared/init-scripts/init.sh b/shared/init-scripts/init.sh index 545d6ee..7889109 100755 --- a/shared/init-scripts/init.sh +++ b/shared/init-scripts/init.sh @@ -10,4 +10,9 @@ createDataStream "logs-frontend-dev" && generate_log_documents 100 | putDocument createDataStream "logs-business-dev" && generate_log_documents 100 | putDocument "logs-business-dev" createDataStream "logs-system-dev" && generate_log_documents 100 | putDocument "logs-system-dev" -createIndex "data-business-index" && generate_log_documents 100 | putDocument "data-business-index" \ No newline at end of file +createIndex "data-business-index" && generate_log_documents 100 | putDocument "data-business-index" + +createKibanaDataView "logs-frontend-*" "Frontend logs" "@timestamp" "admin" "admin" "g1" +createKibanaDataView "logs-business-*" "Business logs" "@timestamp" "admin" "admin" "g1" +createKibanaDataView "logs-system-*" "System logs" "@timestamp" "admin" "admin" "g2" +createKibanaDataView "logs-system-*" "My System logs" "@timestamp" "kibana" "kibana" diff --git a/shared/init-scripts/utils/lib.sh b/shared/init-scripts/utils/lib.sh index bde4c3a..e7f5153 100755 --- a/shared/init-scripts/utils/lib.sh +++ b/shared/init-scripts/utils/lib.sh @@ -102,6 +102,66 @@ function createDataStream() { return 0 } +function createKibanaDataView() { + if [ "$#" -lt 1 ] || [ "$#" -gt 6 ]; then + echo "ERROR: Required: 1) index pattern (title); optionally 2) data view name, 3) time field name, 4) Kibana user, 5) Kibana password, 6) tenancy (ROR group)" + return 1 + fi + + if ! [ -v KIBANA_ADDRESS ] || [ -z "$KIBANA_ADDRESS" ]; then + echo "ERROR: required variable KIBANA_ADDRESS not set or empty" + exit 2 + fi + + INDEX_PATTERN=$1 + DATA_VIEW_NAME=${2:-$INDEX_PATTERN} + TIME_FIELD_NAME=$3 + KBN_USER=${4:-${KIBANA_USER:-}} + KBN_PASS=${5:-${KIBANA_PASSWORD:-}} + TENANCY=$6 + + if [ -z "$KBN_USER" ]; then + echo "ERROR: Kibana user not provided (param 4) and KIBANA_USER env not set" + exit 3 + fi + + if [ -z "$KBN_PASS" ]; then + echo "ERROR: Kibana password not provided (param 5) and KIBANA_PASSWORD env not set" + exit 4 + fi + + data_view_fields="\"title\": \"$INDEX_PATTERN\", \"name\": \"$DATA_VIEW_NAME\"" + if [ -n "$TIME_FIELD_NAME" ]; then + data_view_fields="$data_view_fields, \"timeFieldName\": \"$TIME_FIELD_NAME\"" + fi + + tenancy_header=() + tenancy_info="no tenancy header" + if [ -n "$TENANCY" ]; then + tenancy_header=(-H "x-ror-tenancy-id: $TENANCY") + tenancy_info="tenancy: [$TENANCY]" + fi + + response=$(curl -k -s -L -w "\n%{http_code}" -u "$KBN_USER":"$KBN_PASS" \ + -X POST "$KIBANA_ADDRESS/api/data_views/data_view" \ + -H "Content-Type: application/json" \ + -H "kbn-xsrf: true" \ + "${tenancy_header[@]}" -d "{ + \"data_view\": { $data_view_fields } + }" + ) + + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) + + if [[ "$http_status" != 2* ]]; then + echo "ERROR: Cannot create Kibana data view [$DATA_VIEW_NAME] for index pattern [$INDEX_PATTERN] ($tenancy_info). HTTP status: $http_status, response body: $response_body" + return 5 + fi + + return 0 +} + function putDocument() { if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then echo "ERROR: Required: 1) index name, optionally 2) document JSON string (or via stdin)" diff --git a/xpack-docker-demo-cluster/images/es/Dockerfile b/xpack-docker-demo-cluster/images/es/Dockerfile index bf7ae8a..0a1aa6e 100644 --- a/xpack-docker-demo-cluster/images/es/Dockerfile +++ b/xpack-docker-demo-cluster/images/es/Dockerfile @@ -6,7 +6,6 @@ ARG ES_VERSION USER elasticsearch COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml -COPY conf/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties COPY conf/elastic-certificates.p12 /usr/share/elasticsearch/config/elastic-certificates.p12 RUN echo "" | /usr/share/elasticsearch/bin/elasticsearch-keystore create &&\