Logstash配置文件模板介绍

系统模板介绍

Logstash集群预置了常见的几种Logstash配置文件模板，请参见表1。

redis模板说明

从redis数据库导入数据到Elasticsearch类型集群。

input { 
    redis {
        data_type => "pattern_channel" # data_type, one of ["list", "channel", "pattern_channel"]
        key => "lgs-*" # The name of a Redis list or channel.
        host => "xxx.xxx.xxx.xxxx"
        port => 6379
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
        remove_field => ["@timestamp", "@version"] 
    } 
} 

output { 
    elasticsearch { 
        hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] # host 
        # user => "xxxx" # user name, just for security cluster 
        # password => "xxxx" # password, just for security cluster 
        index => "xxxxxx" # destination _index 
    } 
}

更多信息请参见Logstash官方文档Redis input plugin。

elasticsearch模板说明

从Elasticsearch类型集群导入数据到Elasticsearch类型集群。

input { 
    elasticsearch { 
		# Source ES cluster IP address. When SSL is enabled, please use 7.10.0 and you donot need to add a protocol, otherwise an error is reported.
		hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] 
		# user => "xxxx" # user name, just for security cluster
		# password => "xxxx" # password, just for security cluster
		index => "xxxx,xxx,xxx" # List of indexes to be migrated, separate with commas (,) 
		docinfo => true
		# source ES certificate, the cluster on the cloud this value remains the same. Enter the corresponding path when using a custom certificate. Self-built Logstash cluster, You can download the certs file from the ES cluster details page, enter the corresponding path here. 
		# ca_file => "/rds/datastore/logstash/v7.10.0/package/logstash-7.10.0/extend/certs"  # for 7.10.0
		# ssl => true # Set to true when SSL is enabled. 
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
		remove_field => ["@timestamp", "@version"] 
	} 
} 

output { 
    elasticsearch { 
		# Destination ES cluster IP address. When SSL is enabled, please use 7.10.0 and you donot need to add a protocol.
		hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"]  
		# user => "xxx" # user name, just for security cluster 
		# password => "xxxx" # password, just for security cluster 
		index => "%{[@metadata][_index]}" # destination _index, this config ensure that the index is consistent with that on the source. You can also specify the index name.
		# document_type => "%{[@metadata][_type]}" # destination _type, this config ensure that the document_type is consistent with that on the source. 
		# document_id => "%{[@metadata][_id]}" # destination _id, if you do not need to retain the original _id, delete this line to improve performance. 
		# source ES certificate, the cluster on the cloud this value remains the same. Enter the corresponding path when using a custom certificate. Self-built Logstash cluster, You can download the certs file from the ES cluster details page, enter the corresponding path here. 
		# cacert => "/rds/datastore/logstash/v7.10.0/package/logstash-7.10.0/extend/certs"  # for 7.10.0
		# ssl => true # Set to true when SSL is enabled. 
		# ssl_certificate_verification => false # Set to false to ignore the validation server certificate when SSL is enabled.    
	} 
}

更多信息请参见Logstash官方文档Elasticsearch input plugin。

jdbc模板说明

从jdbc导入数据到Elasticsearch类型集群。

input {
    jdbc {
        # for 7.10.0, jdbc_driver_library => "/rds/datastore/logstash/v7.10.0/package/logstash-7.10.0/extend/jars/mariadb-java-client-2.7.0.jar"
        jdbc_driver_library => "xxxxxxxxxxx"
        jdbc_driver_class => "org.mariadb.jdbc.Driver"
        jdbc_connection_string => "jdbc:mariadb://xxx.xxx.xxx.xxx:xxx/data_base_name" 
        jdbc_user => "xxxx"
        jdbc_password => "xxxx"
        statement => "SELECT * from table_name" # This SQL statement determines the data to be input
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
        remove_field => ["@timestamp", "@version"] 
    } 
} 

output { 
    elasticsearch { 
        hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] # host 
        # user => "xxxx" # user name, just for security cluster 
        # password => "xxxx" # password, just for security cluster 
        index => "xxxxxx" # destination _index 
    } 
}

更多信息请参见Logstash官方文档Jdbc input plugin。

kafka模板说明

从kafka导入数据到Elasticsearch类型集群。

input {
    kafka{
        bootstrap_servers=>"xxx.xxx.xxx.xxx:xxxx"
        topics=>["xxxxxxxx"]
        group_id=>"kafka_es_test"
        auto_offset_reset=>"earliest"
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
        remove_field => ["@timestamp", "@version"] 
    } 
} 

output { 
    elasticsearch { 
        hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] # host 
        # user => "xxxx" # user name, just for security cluster 
        # password => "xxxx" # password, just for security cluster 
        index => "xxxxxx" # destination _index 
    } 
}

更多信息请参见Logstash官方文档Kafka input plugin。

dis模板说明

input {
    dis {
        # For details, see https://support.huaweicloud.com/usermanual-dis/dis_01_0099.html.
        streams => ["YOUR_DIS_STREAM_NAME"]
        endpoint => "https://dis.xxxxx.myhuaweicloud.com" # xxxxx should be replaced by your region name
        ak => "YOUR_ACCESS_KEY_ID"
        sk => "YOUR_SECRET_KEY_ID"
        region => "YOUR_Region"
        project_id => "YOUR_PROJECT_ID"
        group_id => "YOUR_APP_ID"
        client_id => "YOUR_CLIENT_ID"
        auto_offset_reset => "earliest"
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
        remove_field => ["@timestamp", "@version"] 
    } 
} 

output { 
    elasticsearch { 
        hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] # host 
        # user => "xxxx" # user name, just for security cluster 
        # password => "xxxx" # password, just for security cluster 
        index => "xxxxxx" # destination _index 
    } 
}

beats模板说明

input {
    beats {
        port => 5044 # port
    }
}

filter {
    # Delete some fields added by Logstash. 
    mutate {
        remove_field => ["@timestamp", "@version"] 
    } 
} 

output { 
    elasticsearch { 
        hosts => ["http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200", "http://xxx.xxx.xxx.xxx:9200"] # host 
        # user => "xxxx" # user name, just for security cluster 
        # password => "xxxx" # password, just for security cluster 
        index => "xxxxxx" # destination _index 
    } 
}

更多信息请参见Logstash官方文档Beats input plugin。