elasticsearch 8.x configure roles and users for stack monitoring

A lot of changes were made with elasticsearch 8.0 release but the main one that affected me the most was the ability to use the default elastic(superuser role) to access system indices.

According to https://www.elastic.co/guide/en/elasticsearch/reference/current/release-highlights.html#_better_protection_for_system_indices we no longer have write access to system indices with the superuser role.

When configuring stack monitoring using metricbeats this is an issue if you've been just using the elastic user to configure the monitoring. You can add allow_restricted_indices to the role to enable it back however its the lazy and insecure method.

I've summarized the creation of roles and users for metricbeats and filebeats into API calls listed below for easier configuration and reference.

metricbeats - https://www.elastic.co/guide/en/beats/metricbeat/current/feature-roles.htmlmetricbeats

• setup role

PUT _security/role/metricbeat_setup
{
  "cluster": [
    "monitor",
    "manage_ilm"
  ],
  "indices": [
    {
      "names": [
        "metricbeat-*"
      ],
      "privileges": [
        "manage"
      ]
    }
  ]
}

PUT _security/user/metricbeat_setup_user
{
  "password": "test12345",
  "roles": [
    "metricbeat_setup",
    "kibana_admin",
    "ingest_admin"
  ]
}

• monitoring role

PUT _security/role/metricbeat_monitoring
{
  "cluster": [
    "monitor"
  ],
  "indices": [
    {
      "names": [
        ".monitoring-beats-*"
      ],
      "privileges": [
        "create_index",
        "create_doc"
      ]
    }
  ]
}

PUT _security/user/metricbeat_monitoring_user
{
  "password": "test12345",
  "roles": [
    "metricbeat_monitoring",
    "monitoring_user",
    "kibana_admin",
    "remote_monitoring_collector",
    "remote_monitoring_agent"
  ]
}

• writer role

PUT _security/role/metricbeat_writer
{
  "cluster": [
    "monitor",
    "read_ilm"
  ],
  "indices": [
    {
      "names": [
        "metricbeat-*"
      ],
      "privileges": [
        "create_doc",
        "create_index",
        "view_index_metadata"
      ]
    }
  ]
}

PUT _security/user/metricbeat_writer_user
{
  "password": "test12345",
  "roles": [
    "metricbeat_writer"
  ]
}

• reader role

PUT _security/role/metricbeat_reader
{
  "cluster": [],
  "indices": [
    {
      "names": [
        "metricbeat-*"
      ],
      "privileges": [
        "read"
      ]
    }
  ],
  "applications": [
    {
      "application": "kibana-.kibana",
      "privileges": [
        "feature_visualize.all",
        "feature_dashboard.all",
        "feature_discover.all",
        "feature_infrastructure.all"
      ],
      "resources": [
        "*"
      ]
    }
  ]
}

PUT _security/user/metricbeat_reader_user
{
  "password": "test12345",
  "roles": [
    "metricbeat_reader",
    "monitoring_user"
  ]
}

filebeats - https://www.elastic.co/guide/en/beats/filebeat/current/feature-roles.htmlfilebeats

• setup role

PUT _security/role/filebeat_setup
{
  "cluster": [
    "monitor",
    "manage_ilm",
    "manage_ml"
  ],
  "indices": [
    {
      "names": [
        "filebeat-*"
      ],
      "privileges": [
        "manage",
        "read"
      ]
    }
  ]
}

PUT _security/user/filebeat_setup_user
{
  "password": "test12345",
  "roles": [
    "filebeat_setup",
    "kibana_admin",
    "ingest_admin"
  ]
}

• monitoring role

PUT _security/role/filebeat_monitoring
{
  "cluster": [
    "monitor"
  ],
  "indices": [
    {
      "names": [
        ".monitoring-beats-*"
      ],
      "privileges": [
        "create_doc",
        "create_index"
      ]
    }
  ]
}

PUT _security/user/filebeat_monitoring_user
{
  "password": "test12345",
  "roles": [
    "filebeat_monitoring",
    "monitoring_user",
    "kibana_admin",
    "remote_monitoring_collector",
    "remote_monitoring_agent"
  ]
}

• writer role

PUT _security/role/filebeat_writer
{
  "cluster": [
    "monitor",
    "read_pipeline",
    "read_ilm"
  ],
  "indices": [
    {
      "names": [
        "filebeat-*"
      ],
      "privileges": [
        "create_doc",
        "view_index_metadata",
        "create_index"
      ]
    }
  ]
}

PUT _security/user/filebeat_writer_user
{
  "password": "test12345",
  "roles": [
    "filebeat_writer"
  ]
}

• reader role

PUT _security/role/filebeat_reader
{
  "cluster": [],
  "indices": [
    {
      "names": [
        "filebeat-*"
      ],
      "privileges": [
        "read"
      ]
    }
  ],
  "applications": [
    {
      "application": "kibana-.kibana",
      "privileges": [
        "feature_dashboard.all",
        "feature_visualize.all",
        "feature_discover.all",
        "feature_logs.all"
      ],
      "resources": [
        "*"
      ]
    }
  ]
}

PUT _security/user/filebeat_reader_user
{
  "password": "test12345",
  "roles": [
    "filebeat_reader",
    "monitoring_user"
  ]
}

To run the above as curl calls you can convert it to something like

curl -k -u "elastic:PASSWORD" -XPUT "URL:PORT/post-fix" -H 'Content-Type: application/json' -d'
{ BODY OF API CALL}

# example

curl -k -u "elastic:PASSWORD" -XPUT  "https://localhost:9200/_security/role/metricbeat_setup" -H 'Content-Type: application/json' -d'
{
  "cluster": [
    "monitor",
    "manage_ilm"
  ],
  "indices": [
    {
      "names": [
        "metricbeat-*"
      ],
      "privileges": [
        "manage"
      ]
    }
  ]
}

For metricbeat I configured the following:
output.elasticsearch.username: metricbeat_monitoring_user
For kibana/beats/elasticsearch modules
username: remote_monitoring_user

The remote_monitoring_user is a built in user.

For filebeat I configured the following:
output.elasticsearch.username: filebeat_writer_user

However for filebeat I was getting errors:

[elasticsearch.server][ERROR] policy [filebeat] for index [.ds-filebeat-8.0.0-2022.02.12-000001] failed on step [{"phase":"hot","action":"rollover","name":"check-rollover-ready"}]. Moving to ERROR step
org.elasticsearch.ElasticsearchSecurityException: action [indices:admin/rollover] is unauthorized for user [remote_monitoring_user] with roles [remote_monitoring_collector,remote_monitoring_agent] on indices [filebeat-8.0.0,.ds-filebeat-8.0.0-2022.02.12-000001], this action is granted by the index privileges [manage_follow_index,manage,all]

So for the filebeat_writer_user I've updated the role filebeat_writer with

PUT _security/role/filebeat_writer
{
  "cluster": [
    "monitor",
    "read_pipeline",
    "read_ilm",
    "manage"
  ],
  "indices": [
    {
      "names": [
        "filebeat-*"
      ],
      "privileges": [
        "create_doc",
        "view_index_metadata",
        "create_index"
      ]
    },
    {
      "names": [
        "filebeat-*",
        ".ds-filebeat-*"
      ],
      "privileges": [
        "manage",
        "manage_follow_index",
        "all"
      ]
    }
  ]
}