Configure an Elastic Index for the Agent

George Alpizar
George Alpizar
  • Updated

Overview

You can use this document to learn how to set up an elastic index that works with the Edge Delta agent. 


Step 1: Create a Lifecycle Policy

Index lifecycle policies manage indices based on your performance, resiliency, and retention requirements.

Edge Delta provides a simple lifecycle policy, which creates a new index every day and maintains data from the last 15 days. 

  1. In your Elastic's dev console, run the command below. 
    • While you can run the command with the pre-populated settings, you can also change the retention period, as well as other fields.
    • After you run the command, you should have an index lifecycle policy named ed-agent-log-policy.
PUT _ilm/policy/ed-agent-log-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "1d",
            "max_size": "5gb"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "15d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

 


Step 2: Create an Index Template

An index template is useful to configure elastic indices before the indices are created. 

While the Edge Delta agent can be configured to stream various types of observations to the Elasticsearch destination, we recommend that you create the target index with the recommend index template. 

  1. To create the edgedelta elastic index template, in your Elastic's dev console, run the following command. 
    • This action will create an index template named ed-agent-log with field mappings to the ed-agent-log-policy lifecycle policy.

For Elasticsearch 8.x, use the following command:

PUT _index_template/ed-agent-log
{
  "index_patterns": [
    "ed-agent-log-*"
  ],    
      
  "template":{  
    "settings": {
    "number_of_shards": "1",
    "number_of_replicas": "1",
    "lifecycle": {
        "name": "ed-agent-log-policy",
        "rollover_alias": "ed-agent-log"
        },
      "number_of_shards": "1",
      "number_of_replicas": "1"
    },
    "analysis": {
      "analyzer": {
        "custom_label_analyzer": {
          "tokenizer": "custom_label_tokenizer"
        }
      },
      "tokenizer": {
        "custom_label_tokenizer": {
          "type": "pattern",
          "pattern": "\\$"
        }
      }
    }
  },
  "mappings": {
      "_meta": {},
      "_routing": {
        "required": false
      },
      "dynamic": true,
      "numeric_detection": false,
      "date_detection": true,
      "dynamic_date_formats": [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "dynamic_templates": [],
      "properties": {
        "msg": {
          "type": "text"
        },
        "alert_def_id": {
          "type": "keyword"
        },
        "k8s_namespace": {
          "type": "keyword"
        },
        "merge_level": {
          "type": "keyword"
        },
        "ecs_task_family": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "k8s_controller_kind": {
          "type": "keyword"
        },
        "k8s_container_image": {
          "type": "keyword"
        },
        "title": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "type": {
          "type": "keyword"
        },
        "src_name": {
          "type": "keyword"
        },
        "k8s_container_name": {
          "type": "keyword"
        },
        "score": {
          "type": "double"
        },
        "sub_type": {
          "type": "keyword"
        },
        "host": {
          "type": "keyword"
        },
        "capture_flush_mode": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "tag": {
          "type": "keyword"
        },
        "k8s_controller_logical_name": {
          "type": "keyword"
        },
        "timestamp_end": {
          "type": "date"
        },
        "value": {
          "type": "double"
        },
        "timestamp": {
          "index": true,
          "ignore_malformed": false,
          "store": false,
          "type": "date",
          "doc_values": true
        },
        "app": {
          "type": "keyword"
        },
        "capture_size": {
          "coerce": true,
          "index": false,
          "ignore_malformed": false,
          "store": false,
          "type": "long",
          "doc_values": false
        },
        "ecs_task_version": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "index_options": "docs",
          "doc_values": true
        },
        "stat_type": {
          "type": "keyword"
        },
        "docker_container_name": {
          "type": "keyword"
        },
        "conf_id": {
          "type": "keyword"
        },
        "edac_id": {
          "type": "keyword"
        },
        "ip": {
          "type": "ip"
        },
        "k8s_pod_name": {
          "type": "keyword"
        },
        "logical_source": {
          "type": "keyword"
        },
        "environment": {
          "type": "keyword"
        },
        "event_id": {
          "type": "keyword"
        },
        "capture_duration": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "ecs_container": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "capture_bytesize": {
          "coerce": true,
          "index": false,
          "ignore_malformed": false,
          "store": false,
          "type": "long",
          "doc_values": false
        },
        "group_id": {
          "type": "keyword"
        },
        "org_id": {
          "type": "keyword"
        },
        "name": {
          "type": "keyword"
        },
        "alert_def_name": {
          "type": "keyword"
        },
        "ecs_cluster": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "threshold_description": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "threshold_type": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "src_type": {
          "type": "keyword"
        },
        "region": {
          "type": "keyword"
        },
        "custom_labels": {
            "type": "text",
            "analyzer": "custom_label_analyzer"
        },
        "properties": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "docker_image": {
          "type": "keyword"
        }
      }
    }
  }
}

For Elasticsearch 7.x, use the following command:

PUT _template/ed-agent-log?
{
  "order": 0,
  "index_patterns": [
    "ed-agent-log-*"
  ],
  "settings": {
    "index": {
      "lifecycle": {
        "name": "ed-agent-log-policy",
        "rollover_alias": "ed-agent-log"
      },
      "number_of_shards": "1",
      "number_of_replicas": "1"
        },
    "analysis": {
      "analyzer": {
        "custom_label_analyzer": {
          "tokenizer": "custom_label_tokenizer"
        }
      },
      "tokenizer": {
        "custom_label_tokenizer": {
          "type": "pattern",
          "pattern": "\\$"
        }
      }
    }
  },
  "mappings": {
      "_meta": {},
      "_routing": {
        "required": false
      },
      "dynamic": true,
      "numeric_detection": false,
      "date_detection": true,
      "dynamic_date_formats": [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "dynamic_templates": [],
      "properties": {
        "msg": {
          "type": "text"
        },
        "alert_def_id": {
          "type": "keyword"
        },
        "k8s_namespace": {
          "type": "keyword"
        },
        "merge_level": {
          "type": "keyword"
        },
        "ecs_task_family": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "k8s_controller_kind": {
          "type": "keyword"
        },
        "k8s_container_image": {
          "type": "keyword"
        },
        "title": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "type": {
          "type": "keyword"
        },
        "src_name": {
          "type": "keyword"
        },
        "k8s_container_name": {
          "type": "keyword"
        },
        "score": {
          "type": "double"
        },
        "sub_type": {
          "type": "keyword"
        },
        "host": {
          "type": "keyword"
        },
        "capture_flush_mode": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "tag": {
          "type": "keyword"
        },
        "k8s_controller_logical_name": {
          "type": "keyword"
        },
        "timestamp_end": {
          "type": "date"
        },
        "value": {
          "type": "double"
        },
        "timestamp": {
          "index": true,
          "ignore_malformed": false,
          "store": false,
          "type": "date",
          "doc_values": true
        },
        "app": {
          "type": "keyword"
        },
        "capture_size": {
          "coerce": true,
          "index": false,
          "ignore_malformed": false,
          "store": false,
          "type": "long",
          "doc_values": false
        },
        "ecs_task_version": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "index_options": "docs",
          "doc_values": true
        },
        "stat_type": {
          "type": "keyword"
        },
        "docker_container_name": {
          "type": "keyword"
        },
        "conf_id": {
          "type": "keyword"
        },
        "edac_id": {
          "type": "keyword"
        },
        "ip": {
          "type": "ip"
        },
        "k8s_pod_name": {
          "type": "keyword"
        },
        "logical_source": {
          "type": "keyword"
        },
        "environment": {
          "type": "keyword"
        },
        "event_id": {
          "type": "keyword"
        },
        "capture_duration": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "ecs_container": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "capture_bytesize": {
          "coerce": true,
          "index": false,
          "ignore_malformed": false,
          "store": false,
          "type": "long",
          "doc_values": false
        },
        "group_id": {
          "type": "keyword"
        },
        "org_id": {
          "type": "keyword"
        },
        "name": {
          "type": "keyword"
        },
        "alert_def_name": {
          "type": "keyword"
        },
        "ecs_cluster": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": true,
          "store": false,
          "type": "keyword",
          "index_options": "docs",
          "split_queries_on_whitespace": false,
          "doc_values": true
        },
        "threshold_description": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "threshold_type": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "src_type": {
          "type": "keyword"
        },
        "region": {
          "type": "keyword"
        },
        "custom_labels": {
            "type": "text",
            "analyzer": "custom_label_analyzer"
        },
        "properties": {
          "eager_global_ordinals": false,
          "norms": false,
          "index": false,
          "store": false,
          "type": "keyword",
          "split_queries_on_whitespace": false,
          "doc_values": false
        },
        "docker_image": {
          "type": "keyword"
        }
      }
    }
  }

Step 3: Create the First Index 

To generate a daily index, you must create the first index. 

This first index will inherit field mappings and policies from the template. 

  1. In your Elastic's dev console, run the following command. 
      • For Elasticsearch 8.x, use the following command:
        PUT ed-index
        {
         "aliases": {
           "ed-agent-log": {
             "is_write_index": true
           }
         }
        }
      • For Elasticsearch 7.x, use the following command:
        PUT /%3Ced-agent-log-%7Bnow%2Fd%7D-000001%3E
        {
         "aliases": {
           "ed-agent-log": {
             "is_write_index": true
           }
         }
        }
  2. Access the Index Management > Indices, and then locate a new index, similar to ed-agent-log-2020.10.22-000000 with the current date. 

Your Elastic environment is ready for Edge Delta. You can deploy an agent with an Elastic destination that points to the ed-agent-log index.

To learn how to deploy the agent, see:

 


Share this document