[RORDEV-1069] reproducer

ror-demo-cluster/conf/es/readonlyrest.yml

@@ -18,18 +18,26 @@ readonlyrest:
 
     - name: "End users"
       groups: ["EndUsers"]
-      indices: ["frontend_logs", "kibana_sample_data_*"]
+      indices: ["frontend_logs*", "kibana_sample_data_*"]
       kibana:
         index: .kibana_end_@{user}
         access: rw
         hide_apps: ["Security", "Observability"]
 
     - name: "Business users"
       groups: ["BusinessUsers"]
-      indices: ["business_logs", "kibana_sample_data_*"]
+      indices: ["business_logs*", "kibana_sample_data_*"]
       kibana:
         index: .kibana_business_@{user}
-        access: rw
+        access: ro
+        hide_apps: ["Security", "Observability"]
+
+    - name: "Test users"
+      groups: ["TestUsers"]
+      indices: ["business_logs*", "kibana_sample_data_*"]
+      kibana:
+        index: .kibana_test_@{user}
+        access: ro_strict
         hide_apps: ["Security", "Observability"]
 
   users:
@@ -42,6 +50,8 @@ readonlyrest:
           name: "End Users"
         - id: "BusinessUsers"
           name: "Business Users"
+        - id: "TestUsers"
+          name: "Test Users"
 
     - username: user1
       auth_key: user1:test
@@ -50,26 +60,11 @@ readonlyrest:
           name: "End Users"
         - id: "BusinessUsers"
           name: "Business Users"
+        - id: "TestUsers"
+          name: "Test Users"
 
     - username: user2
       auth_key: user2:test
       groups:
         - id: "EndUsers"
           name: "End Users"
-
-    - username: "*"
-      ror_kbn_auth:
-        name: "kbn1"
-        groups: ["*"]
-      groups:
-        - local_group:
-            id: "EndUsers"
-            name: "End Users"
-          external_group_ids: [ "extEndUsers" ]
-        - local_group:
-            id: "BusinessUsers"
-            name: "Business Users"
-          external_group_ids: [ "extBusinessUsers" ]
-  ror_kbn:
-    - name: kbn1
-      signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf"

ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml

@@ -12,53 +12,15 @@ server.ssl.certificate: /usr/share/kibana/config/kibana.crt
 server.ssl.key: /usr/share/kibana/config/kibana.key
 server.ssl.redirectHttpFromPort: 80
 
-readonlyrest_kbn.logLevel: info
-readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm'
-readonlyrest_kbn:
-  auth:
-      signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf"
-      saml_keycloak:
-        buttonName: 'Keycloak SAML'
-        enabled: true
-        type: 'saml'
-        issuer: 'ror-saml'
-        entryPoint: 'http://kc.localhost:8080/realms/ror/protocol/saml'
-        kibanaExternalHost: 'localhost:15601'
-        protocol: 'https'
-        usernameParameter: 'nameID'
-        groupsParameter: 'Role'
-        logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/saml'
-        YOU_SHOULD_READ_ME_IN_STRATEGY_CONFIGURATION_LOG: 'unknown conf params should be passed unmodified to the underlying passport-saml library'
-        cert: 'MIICrDCCAZQCCQDN5Tcc+Rn6rTANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1rZXljbG9hay1zYW1sMB4XDTI1MTExMDA0MjQyOFoXDTM1MTEwODA0MjQyOFowGDEWMBQGA1UEAwwNa2V5Y2xvYWstc2FtbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv0EFcTjNkpn5kV/XedCJ/AQKsPafZ7u33u3zfYgZbTh0V+CJ4bUAZyTfvGrcxR/iZy3hkYQBL7DGM6c/fYJGq1IH+/jxAk/GBY9tVnxotRsyhIMJYtZCb+DBUVX+wLowv2kZPlf/6OibjVpy+I6klQxIU8aeMkd+M/Phl97l+pRUjTuQZvEaEtkVLcsG6gEUcNo2X0MpNFwT7UgpGZLGy8zSHGNu703tb8w0oCYTEj42WqLWYNm5NMqD/clbPRj8g+1qZHpSoIH+p/cqVfU+oAZd3R2Y8SGR2OhYwyu9wHRnuqgiEgCIYbzwyh3IzX+57R8MQxHnjBFnzJKFtBWHMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAUGrj7hEEHJvm58RYTjqhcJ38MVkgEmAnGovObxvS9Xi6ZBct4irWdpyESowRJtY4nxFGs0uiArOpQDkteSeDzJs/IkSg3xTx3UwOpevf8IV5gU8Bwq68Fyh9An8NvczE5XhLZD1Tacphlz5OzoXthMsS1pcCumr5ZELwMtvLYAWkEV/cKmlis4JuMEZXc5v9KVybsmHv1hgM+fxg1neuWrPK1JebuVm4oaHUHYKCxqn9MXjnDvOq6MkYtKBfcYf6BKk29lapYuHNiRTi0hMPD1tWVaZg3H1/uMxlLXDxAZqkS4DNq/7MeIMUWemOqxQHZAi4rtplrVl/F3WES6pqWw=='
-      oidc_keycloak:
-        buttonName: "Keycloak OIDC"
-        type: "oidc"
-        protocol: "https"
-        issuer: 'http://kc.localhost:8080/realms/ror'
-        authorizationURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/auth'
-        tokenURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/token'
-        userInfoURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/userinfo'
-        jwksURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/certs'
-        clientID: 'ror-oidc'
-        clientSecret: 'kibanasecret123'
-        scope: 'openid profile email'
-        usernameParameter: 'preferred_username'
-        groupsParameter: 'groups'
-        kibanaExternalHost: 'localhost:15601'
-        logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/logout'
-      oidc_lemon_ldap:
-        buttonName: "LemonLDAP OpenID"
-        type: "oidc"
-        protocol: "https"
-        issuer: 'https://oidctest.wsweet.org/'
-        authorizationURL: 'https://oidctest.wsweet.org/oauth2/authorize'
-        tokenURL: 'https://oidctest.wsweet.org/oauth2/token'
-        userInfoURL: 'https://oidctest.wsweet.org/oauth2/userinfo'
-        clientID: 'private'
-        clientSecret: 'tardis'
-        scope: 'openid users roles'
-        usernameParameter: 'sub'
-        groupsParameter: 'roles'
-        kibanaExternalHost: 'localhost:15601'
-        logoutUrl: 'https://oidctest.wsweet.org/oauth2/logout'
-        jwksURL: 'https://oidctest.wsweet.org/oauth2/jwks'
+xpack.encryptedSavedObjects:
+  encryptionKey: "min-32-byte-long-strong-encryption-key"
+
+logging:
+  root:
+    level: debug
+  loggers:
+  - name: elasticsearch
+    level: trace
+
+readonlyrest_kbn.logLevel: trace
+readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm'

ror-demo-cluster/docker-compose.yml

@@ -1,27 +1,4 @@
 services:
-  # Enterprise-only service
-  keycloak:
-    image: quay.io/keycloak/keycloak:20.0.5
-    profiles: ["ENT"]
-    environment:
-      - KEYCLOAK_ADMIN=admin
-      - KEYCLOAK_ADMIN_PASSWORD=admin
-      - KEYCLOAK_FRONTEND_URL=http://kc.localhost:8080
-    volumes:
-      - ./conf/keycloak/ror-realm.json:/opt/keycloak/data/import/ror-realm.json:ro
-    command: ["start-dev", "--import-realm", "--hostname=kc.localhost", "--http-enabled=true", "--http-port=8080"]
-    ports:
-      - "8080:8080"
-    healthcheck:
-      test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 5 --retry-connrefused http://127.0.0.1:8080/realms/ror/.well-known/openid-configuration >/dev/null || exit 1"]
-      interval: 10s
-      timeout: 10s
-      retries: 30
-      start_period: 40s
-    networks:
-      es-ror-network:
-        aliases:
-          - kc.localhost
 
   es-ror:
     build:
@@ -55,6 +32,15 @@ services:
         soft: -1
         hard: -1
 
+  es-ror-proxy:
+    image: mitmproxy/mitmproxy
+    command:
+      mitmdump --set validate_inbound_headers=false --mode reverse:https://es-ror:9200 --ssl-insecure --verbose
+    depends_on:
+      - es-ror
+    networks:
+      - es-ror-network
+
   kbn-ror:
     build:
       context: .
@@ -67,13 +53,10 @@ services:
     depends_on:
       es-ror:
         condition: service_healthy
-      keycloak:
-        condition: service_healthy
-        required: false
     ports:
       - "15601:5601"
     environment:
-      ELASTICSEARCH_HOSTS: https://es-ror:9200
+      ELASTICSEARCH_HOSTS: https://es-ror-proxy:8080
       ROR_ACTIVATION_KEY: $ROR_ACTIVATION_KEY
     healthcheck:
       test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 2 --retry-connrefused -u admin:admin https://127.0.0.1:5601/api/features >/dev/null || exit 1"]

shared/init-scripts/init.sh

@@ -4,37 +4,10 @@ cd "$(dirname "$0")"
 
 source utils/lib.sh
 
-function generate_log_documents() {
-  if [ "$#" -ne 1 ]; then
-    echo "ERROR: One required: 1) number of documents to generate"
-    return 1
-  fi
+#createDataStream "frontend_logs_ds" && generate_log_documents 100 | putDocument "frontend_logs_ds"
+#createDataStream "business_logs_ds" && generate_log_documents 50 | putDocument "business_logs_ds"
+#createDataStream "system_logs_ds" && generate_log_documents 60 | putDocument "system_logs_ds"
 
-  N=$1 
-
-  for ((i = 1; i <= N; i++)); do
-    user_id=$((RANDOM % 10000 + 1))
-    timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") 
-    log_message="User $user_id login successful"
-    level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")"
-
-    echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }"
-  done
-}
-
-function index_documents() {
-   if [ "$#" -ne 1 ]; then
-    echo "ERROR: One required: 1) index name"
-    return 1
-  fi
-
-  INDEX_NAME=$1 
-
-  while IFS= read -r document; do
-    putDocument "$INDEX_NAME" "$document"
-  done
-}
-
-generate_log_documents 100 | index_documents "frontend_logs"
-generate_log_documents 50 | index_documents "business_logs"
-generate_log_documents 60 | index_documents "system_logs"
+#createIndex "frontend_logs_index" && generate_log_documents 100 | putDocument "frontend_logs_index"
+#createIndex "business_logs_index" && generate_log_documents 50 | putDocument "business_logs_index"
+#createIndex "system_logs_index" && generate_log_documents 60 | putDocument "system_logs_index"

shared/init-scripts/utils/lib.sh

@@ -7,9 +7,92 @@ function pick_randomly() {
   echo "${OPTIONS[$RANDOM_INDEX]}"
 }
 
+function createIndex() {
+  if [ "$#" -ne 1 ]; then
+    echo "ERROR: One parameter required: 1) index name"
+    return 1
+  fi
+
+  if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty"
+    exit 2
+  fi
+
+  if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_USER not set or empty"
+    exit 3
+  fi
+
+  if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty"
+    exit 4
+  fi
+
+  INDEX_NAME=$1
+
+  response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \
+    -X PUT "$ELASTICSEARCH_ADDRESS/$INDEX_NAME" \
+    -H "Content-Type: application/json"
+  )
+
+  http_status=$(echo "$response" | tail -n 1)
+  response_body=$(echo "$response" | sed \$d)
+
+  if [[ "$http_status" != 2* ]]; then
+    echo "ERROR: Cannot create index [$INDEX_NAME]. HTTP status: $http_status, response body: $response_body"
+    return 5
+  fi
+
+  return 0
+}
+
+function createDataStream() {
+  if [ "$#" -ne 1 ]; then
+    echo "ERROR: One parameter required: 1) data stream name"
+    return 1
+  fi
+
+  if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty"
+    exit 2
+  fi
+
+  if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_USER not set or empty"
+    exit 3
+  fi
+
+  if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then
+    echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty"
+    exit 4
+  fi
+
+  STREAM_NAME=$1
+  TEMPLATE_NAME="${STREAM_NAME}-template"
+
+  response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \
+    -X PUT "$ELASTICSEARCH_ADDRESS/_index_template/$TEMPLATE_NAME" \
+    -H "Content-Type: application/json" -d "{
+      \"index_patterns\": [\"$STREAM_NAME\"],
+      \"data_stream\": {},
+      \"priority\": 500
+    }"
+  )
+
+  http_status=$(echo "$response" | tail -n 1)
+  response_body=$(echo "$response" | sed \$d)
+
+  if [[ "$http_status" != 2* ]]; then
+    echo "ERROR: Cannot create index template for data stream [$STREAM_NAME]. HTTP status: $http_status, response body: $response_body"
+    return 5
+  fi
+
+  return 0
+}
+
 function putDocument() {
-  if [ "$#" -ne 2 ]; then
-    echo "ERROR: Three parameters required: 1) index name, 2) document JSON string"
+  if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
+    echo "ERROR: Required: 1) index name, optionally 2) document JSON string (or via stdin)"
     return 1
   fi
 
@@ -29,22 +112,50 @@ function putDocument() {
   fi
 
   INDEX_NAME=$1
-  DOCUMENT_CONTENT=$2
 
-set -x
+  if [ "$#" -eq 2 ]; then
+    putSingleDocument "$INDEX_NAME" "$2"
+  else
+    while IFS= read -r DOCUMENT_CONTENT; do
+      putSingleDocument "$INDEX_NAME" "$DOCUMENT_CONTENT"
+    done
+  fi
+}
 
-  respone=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \
+function putSingleDocument() {
+  INDEX_NAME=$1
+  DOCUMENT_CONTENT=$2
+
+  response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \
     -X POST "$ELASTICSEARCH_ADDRESS/$INDEX_NAME/_doc/" \
     -H "Content-Type: application/json" -d "$DOCUMENT_CONTENT"
   )
 
-  http_status=$(echo "$respone" | tail -n 1)
-  response_body=$(echo "$respone" | sed \$d)
+  http_status=$(echo "$response" | tail -n 1)
+  response_body=$(echo "$response" | sed \$d)
 
   if [[ "$http_status" != 2* ]] ; then
-    echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY"
+    echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $http_status, response body: $response_body"
     return 5
   fi
 
   return 0
 }
+
+function generate_log_documents() {
+  if [ "$#" -ne 1 ]; then
+    echo "ERROR: One required: 1) number of documents to generate"
+    return 1
+  fi
+
+  N=$1
+
+  for ((i = 1; i <= N; i++)); do
+    user_id=$((RANDOM % 10000 + 1))
+    timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    log_message="User $user_id login successful"
+    level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")"
+
+    echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"@timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }"
+  done
+}