[RORDEV-2053] analysis

examples/ror-with-kibana-reverse-proxy-demo/docker-compose.yml

@@ -124,6 +124,7 @@ services:
       ELASTICSEARCH_ADDRESS: https://es-ror:9200
       ELASTICSEARCH_USER: kibana
       ELASTICSEARCH_PASSWORD: kibana
+      KIBANA_ADDRESS: https://kbn-ror:5601
     healthcheck:
       test: "test -f /tmp/init_done || exit 1"
       interval: 10s

ror-demo-cluster/.env-showcase

@@ -5,10 +5,12 @@
 #   Dockerfile-use-ror-binaries-from-api  - download ROR plugin from API (requires ROR_ES_VERSION / ROR_KBN_VERSION)
 #   Dockerfile-use-ror-binaries-from-file - use a local plugin file (requires ES_ROR_FILE / KBN_ROR_FILE)
 
-#ES_VERSION=8.19.11
-#ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-file
-#ES_ROR_FILE=readonlyrest-1.69.0-pre01_es8.19.11.zip
+ES_VERSION=8.19.11
+#ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-api
+#ROR_ES_VERSION=1.69.1
+ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-file
+ES_ROR_FILE=readonlyrest-1.70.0-pre10_es8.19.11.zip
 
-#KBN_VERSION=8.19.11
-#KBN_DOCKERFILE=Dockerfile-use-ror-binaries-from-api
-#ROR_KBN_VERSION=1.68.0
+KBN_VERSION=8.19.11
+KBN_DOCKERFILE=Dockerfile-use-ror-binaries-from-file
+KBN_ROR_FILE=readonlyrest_kbn_universal-1.70.0-pre13_es8.19.11.zip

ror-demo-cluster/conf/es/readonlyrest.yml

@@ -11,66 +11,48 @@ readonlyrest:
       auth_key: kibana:kibana
       verbosity: error
 
-    - name: "Admins"
-      groups: [Administrators]
+    - name: "admin"
+      type: allow
+      users: ["admin"]
+      groups: ["*"]
       kibana:
-        access: admin
+        access: unrestricted
+        index: .kibana_@{acl:current_group}
+        hide_apps: ["app1"]
+
+    - name: "Group1 users"
+      type: allow
+      groups_or: ["g1"]
+      kibana:
+        access: rw
+        index: .kibana_@{acl:current_group}
+        hide_apps: ["app2"]
 
-    - name: "End users"
-      groups: ["EndUsers"]
-      indices: ["*-frontend-*", "kibana_sample_data_*"]
+    - name: "Group2 users"
+      type: allow
+      groups_or: ["g2"]
       kibana:
-        index: .kibana_end_@{user}
         access: rw
-        hide_apps: ["Security", "Observability"]
+        index: .kibana_@{acl:current_group}
+        hide_apps: ["app3"]
 
-    - name: "Business users"
-      groups: ["BusinessUsers"]
-      indices: ["*-business-*", "kibana_sample_data_*"]
+    - name: "NO tenancy user"
+      type: allow
+      auth_key: user3:test
       kibana:
-        index: .kibana_business_@{user}
-        access: ro
-        hide_apps: ["Security", "Observability"]
+        access: rw
+        # default kibana index
+        hide_apps: ["app4"]
 
   users:
     - username: admin
       auth_key: admin:admin
-      groups:
-        - id: "Administrators"
-          name: "Administrators"
-        - id: "EndUsers"
-          name: "End Users"
-        - id: "BusinessUsers"
-          name: "Business Users"
+      groups: [admin, g1, g2]
 
     - username: user1
       auth_key: user1:test
-      groups:
-        - id: "EndUsers"
-          name: "End Users"
-        - id: "BusinessUsers"
-          name: "Business Users"
+      groups: [g1, g2]
 
     - username: user2
       auth_key: user2:test
-      groups:
-        - id: "EndUsers"
-          name: "End Users"
-
-    - username: "*"
-      ror_kbn_auth:
-        name: "kbn1"
-        groups: ["*"]
-      groups:
-        - local_group:
-            id: "EndUsers"
-            name: "End Users"
-          external_group_ids: [ "extEndUsers" ]
-        - local_group:
-            id: "BusinessUsers"
-            name: "Business Users"
-          external_group_ids: [ "extBusinessUsers" ]
-
-  ror_kbn:
-    - name: kbn1
-      signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf"
+      groups: [g1]

ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml

@@ -14,53 +14,5 @@ server.ssl.redirectHttpFromPort: 80
 
 xpack.encryptedSavedObjects.encryptionKey: "min-32-byte-long-strong-encryption-key"
 
-readonlyrest_kbn.logLevel: info
+readonlyrest_kbn.logLevel: trace
 readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm'
-readonlyrest_kbn:
-  auth:
-      signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf"
-      saml_keycloak:
-        buttonName: 'Keycloak SAML'
-        enabled: true
-        type: 'saml'
-        issuer: 'ror-saml'
-        entryPoint: 'http://kc.localhost:8080/realms/ror/protocol/saml'
-        kibanaExternalHost: 'localhost:15601'
-        protocol: 'https'
-        usernameParameter: 'nameID'
-        groupsParameter: 'Role'
-        logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/saml'
-        YOU_SHOULD_READ_ME_IN_STRATEGY_CONFIGURATION_LOG: 'unknown conf params should be passed unmodified to the underlying passport-saml library'
-        cert: 'MIICrDCCAZQCCQDN5Tcc+Rn6rTANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1rZXljbG9hay1zYW1sMB4XDTI1MTExMDA0MjQyOFoXDTM1MTEwODA0MjQyOFowGDEWMBQGA1UEAwwNa2V5Y2xvYWstc2FtbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv0EFcTjNkpn5kV/XedCJ/AQKsPafZ7u33u3zfYgZbTh0V+CJ4bUAZyTfvGrcxR/iZy3hkYQBL7DGM6c/fYJGq1IH+/jxAk/GBY9tVnxotRsyhIMJYtZCb+DBUVX+wLowv2kZPlf/6OibjVpy+I6klQxIU8aeMkd+M/Phl97l+pRUjTuQZvEaEtkVLcsG6gEUcNo2X0MpNFwT7UgpGZLGy8zSHGNu703tb8w0oCYTEj42WqLWYNm5NMqD/clbPRj8g+1qZHpSoIH+p/cqVfU+oAZd3R2Y8SGR2OhYwyu9wHRnuqgiEgCIYbzwyh3IzX+57R8MQxHnjBFnzJKFtBWHMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAUGrj7hEEHJvm58RYTjqhcJ38MVkgEmAnGovObxvS9Xi6ZBct4irWdpyESowRJtY4nxFGs0uiArOpQDkteSeDzJs/IkSg3xTx3UwOpevf8IV5gU8Bwq68Fyh9An8NvczE5XhLZD1Tacphlz5OzoXthMsS1pcCumr5ZELwMtvLYAWkEV/cKmlis4JuMEZXc5v9KVybsmHv1hgM+fxg1neuWrPK1JebuVm4oaHUHYKCxqn9MXjnDvOq6MkYtKBfcYf6BKk29lapYuHNiRTi0hMPD1tWVaZg3H1/uMxlLXDxAZqkS4DNq/7MeIMUWemOqxQHZAi4rtplrVl/F3WES6pqWw=='
-      oidc_keycloak:
-        buttonName: "Keycloak OIDC"
-        type: "oidc"
-        protocol: "https"
-        issuer: 'http://kc.localhost:8080/realms/ror'
-        authorizationURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/auth'
-        tokenURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/token'
-        userInfoURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/userinfo'
-        jwksURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/certs'
-        clientID: 'ror-oidc'
-        clientSecret: 'kibanasecret123'
-        scope: 'openid profile email'
-        usernameParameter: 'preferred_username'
-        groupsParameter: 'groups'
-        kibanaExternalHost: 'localhost:15601'
-        logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/logout'
-      oidc_lemon_ldap:
-        buttonName: "LemonLDAP OpenID"
-        type: "oidc"
-        protocol: "https"
-        issuer: 'https://oidctest.wsweet.org/'
-        authorizationURL: 'https://oidctest.wsweet.org/oauth2/authorize'
-        tokenURL: 'https://oidctest.wsweet.org/oauth2/token'
-        userInfoURL: 'https://oidctest.wsweet.org/oauth2/userinfo'
-        clientID: 'private'
-        clientSecret: 'tardis'
-        scope: 'openid users roles'
-        usernameParameter: 'sub'
-        groupsParameter: 'roles'
-        kibanaExternalHost: 'localhost:15601'
-        logoutUrl: 'https://oidctest.wsweet.org/oauth2/logout'
-        jwksURL: 'https://oidctest.wsweet.org/oauth2/jwks'

ror-demo-cluster/docker-compose.yml

@@ -40,7 +40,7 @@ services:
       - node.name=es-ror-single
       - discovery.type=single-node
       - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=1 -Dcom.readonlyrest.settings.loading.delay=0s"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=1 -Dcom.readonlyrest.settings.loading.delay=1"
       - ES_VERSION=${ES_VERSION:-ES_VERSION_NOT_CONFIGURED}
     healthcheck:
       test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 2 --retry-connrefused -u admin:admin https://127.0.0.1:9200/_cluster/health >/dev/null || exit 1"]
@@ -101,6 +101,7 @@ services:
       ELASTICSEARCH_ADDRESS: https://es-ror:9200
       ELASTICSEARCH_USER: kibana
       ELASTICSEARCH_PASSWORD: kibana
+      KIBANA_ADDRESS: https://kbn-ror:5601
     healthcheck:
       test: "test -f /tmp/init_done || exit 1"
       interval: 10s

shared/init-scripts/init.sh

@@ -10,4 +10,9 @@ createDataStream "logs-frontend-dev" && generate_log_documents 100 | putDocument
 createDataStream "logs-business-dev" && generate_log_documents 100 | putDocument "logs-business-dev"
 createDataStream "logs-system-dev" && generate_log_documents 100 | putDocument "logs-system-dev"
 
-createIndex "data-business-index" && generate_log_documents 100 | putDocument "data-business-index"
+createIndex "data-business-index" && generate_log_documents 100 | putDocument "data-business-index"
+
+createKibanaDataView "logs-frontend-*" "Frontend logs" "@timestamp" "admin" "admin" "g1"
+createKibanaDataView "logs-business-*" "Business logs" "@timestamp" "admin" "admin" "g1"
+createKibanaDataView "logs-system-*" "System logs" "@timestamp" "admin" "admin" "g2"
+createKibanaDataView "logs-system-*" "My System logs" "@timestamp" "kibana" "kibana"

shared/init-scripts/utils/lib.sh

@@ -102,6 +102,66 @@ function createDataStream() {
   return 0
 }
 
+function createKibanaDataView() {
+  if [ "$#" -lt 1 ] || [ "$#" -gt 6 ]; then
+    echo "ERROR: Required: 1) index pattern (title); optionally 2) data view name, 3) time field name, 4) Kibana user, 5) Kibana password, 6) tenancy (ROR group)"
+    return 1
+  fi
+
+  if ! [ -v KIBANA_ADDRESS ] || [ -z "$KIBANA_ADDRESS" ]; then
+    echo "ERROR: required variable KIBANA_ADDRESS not set or empty"
+    exit 2
+  fi
+
+  INDEX_PATTERN=$1
+  DATA_VIEW_NAME=${2:-$INDEX_PATTERN}
+  TIME_FIELD_NAME=$3
+  KBN_USER=${4:-${KIBANA_USER:-}}
+  KBN_PASS=${5:-${KIBANA_PASSWORD:-}}
+  TENANCY=$6
+
+  if [ -z "$KBN_USER" ]; then
+    echo "ERROR: Kibana user not provided (param 4) and KIBANA_USER env not set"
+    exit 3
+  fi
+
+  if [ -z "$KBN_PASS" ]; then
+    echo "ERROR: Kibana password not provided (param 5) and KIBANA_PASSWORD env not set"
+    exit 4
+  fi
+
+  data_view_fields="\"title\": \"$INDEX_PATTERN\", \"name\": \"$DATA_VIEW_NAME\""
+  if [ -n "$TIME_FIELD_NAME" ]; then
+    data_view_fields="$data_view_fields, \"timeFieldName\": \"$TIME_FIELD_NAME\""
+  fi
+
+  tenancy_header=()
+  tenancy_info="no tenancy header"
+  if [ -n "$TENANCY" ]; then
+    tenancy_header=(-H "x-ror-tenancy-id: $TENANCY")
+    tenancy_info="tenancy: [$TENANCY]"
+  fi
+
+  response=$(curl -k -s -L -w "\n%{http_code}" -u "$KBN_USER":"$KBN_PASS" \
+    -X POST "$KIBANA_ADDRESS/api/data_views/data_view" \
+    -H "Content-Type: application/json" \
+    -H "kbn-xsrf: true" \
+    "${tenancy_header[@]}" -d "{
+      \"data_view\": { $data_view_fields }
+    }"
+  )
+
+  http_status=$(echo "$response" | tail -n 1)
+  response_body=$(echo "$response" | sed \$d)
+
+  if [[ "$http_status" != 2* ]]; then
+    echo "ERROR: Cannot create Kibana data view [$DATA_VIEW_NAME] for index pattern [$INDEX_PATTERN] ($tenancy_info). HTTP status: $http_status, response body: $response_body"
+    return 5
+  fi
+
+  return 0
+}
+
 function putDocument() {
   if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
     echo "ERROR: Required: 1) index name, optionally 2) document JSON string (or via stdin)"

xpack-docker-demo-cluster/images/es/Dockerfile

@@ -6,7 +6,6 @@ ARG ES_VERSION
 
 USER elasticsearch
 COPY conf/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
-COPY conf/log4j2.properties /usr/share/elasticsearch/config/log4j2.properties
 COPY conf/elastic-certificates.p12 /usr/share/elasticsearch/config/elastic-certificates.p12
 
 RUN echo "" | /usr/share/elasticsearch/bin/elasticsearch-keystore create &&\