ElastAlert elasticsearch elk

docker-compose

version: '3.3'

services:
  elasticsearch:
    build:
      context: elasticsearch/
    volumes:
      #- ./elasticsearch/esdata:/usr/share/elasticsearch/data:rw
      - alldata:/usr/share/elasticsearch/data:rw
      #- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    networks:
      - fastdev

  logstash:
    build:
      context: logstash/
    volumes:
      - ./logstash/config/logstash.yml:/etc/logstash/logstash.yml:ro
      - ./logstash/pipeline:/etc/logstash/conf.d:ro
    ports:
      - "5000:5000"
      - "5044:5044"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - fastdev
    depends_on:
      - elasticsearch

  kibana:
    build:
      context: kibana/
    volumes:
      - ./kibana/config/:/usr/share/kibana/config:ro
    ports:
      - "5601:5601"
    networks:
      - fastdev
    depends_on:
      - elasticsearch

  fakelog:
    build:
      context: fakelog/
    volumes:
      - ./fakelog/logs:/ethereum-etl/output/:rw
    networks:
      - fastdev

  elastalert:
    build:
      context: elastalert/
    volumes:
      - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json:ro
      - ./elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml:ro
      - ./elastalert/rules:/opt/elastalert/rules:rw
      - ./elastalert/rule_templates:/opt/elastalert/rule_templates:ro
    ports:
      - "3030:3030"
    #  - "3333:3333"
    environment:
      - net=elasticsearch
    networks:
      - fastdev

volumes:  
  alldata: 

networks:
  fastdev:
    driver: bridge

Only elasticsearch/Dockerfile some error happen because debian mirror. Other YAML see before blogs

FROM java:8-jre

# grab gosu for easy step-down from root
ENV GOSU_VERSION 1.7
RUN set -x \
 && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
 && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
 && export GNUPGHOME="$(mktemp -d)" \
 && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
 && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
 && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \
 && chmod +x /usr/local/bin/gosu \
 && gosu nobody true

# https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html
# https://packages.elasticsearch.org/GPG-KEY-elasticsearch
#RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4

#ENV ELASTICSEARCH_MAJOR 1.7
#ENV ELASTICSEARCH_VERSION 1.7.5
#ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/1.7/debian

#RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list

#RUN set -x \
# && apt-get clean && apt-get update \
# && apt-get install -y --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \
# && rm -rf /var/lib/apt/lists/*

RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -
RUN echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list
RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie.list
RUN echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list
RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list
RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
RUN apt-get -o Acquire::Check-Valid-Until=false update && apt-get install elasticsearch

ENV PATH /usr/share/elasticsearch/bin:$PATH

WORKDIR /usr/share/elasticsearch

RUN set -ex \
 && for path in \
  ./data \
  ./logs \
  ./config \
  ./config/scripts \
 ; do \
  mkdir -p "$path"; \
  chown -R elasticsearch:elasticsearch "$path"; \
 done

COPY config ./config

#VOLUME /usr/share/elasticsearch/data

COPY docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh

EXPOSE 9200 9300
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["elasticsearch"]

ElastAlert/Dockerfile

FROM bitsensor/elastalert:latest

ElastAlert/config/config.json

{
  "appName": "elastalert-server",
  "port": 3030,
  "wsport": 3333,
  "elastalertPath": "/opt/elastalert",
  "verbose": false,
  "es_debug": false,
  "debug": false,
  "rulesPath": {
    "relative": true,
    "path": "/rules"
  },
  "templatesPath": {
    "relative": true,
    "path": "/rule_templates"
  },
  "es_host": "elasticsearch",
  "es_port": 9200,
  "writeback_index": "elastalert_status"
}

ElastAlert/config/elastalert.yaml

es_host: elasticsearch
es_port: 9200
rules_folder: rules

# How often ElastAlert will query elasticsearch                
# The unit can be anything from weeks to seconds
run_every:
  seconds: 5
  #minutes: 1
# ElastAlert will buffer results from the most recent                                                      
# period of time, in case some log sources are not in real time
buffer_time:
  minutes: 15
# If an alert fails for some reason, ElastAlert will retry                                                 
# sending the alert until this time period has elapsed 
alert_time_limit:
  minutes: 2
  #days: 2
  
writeback_index: elastalert_status

#smtp_host: ""

ElastAlert/config/rules/test_metric.yaml

name: Example rule

# How often ElastAlert will query elasticsearch                
# The unit can be anything from weeks to seconds
run_every:
  seconds: 5
  #minutes: 1
# ElastAlert will buffer results from the most recent                                                      
# period of time, in case some log sources are not in real time
buffer_time:
  minutes: 15
# If an alert fails for some reason, ElastAlert will retry                                                 
# sending the alert until this time period has elapsed 
alert_time_limit:
  minutes: 2
  #days: 2
    
type: metric_aggregation

index: filebeat-*

metric_agg_key: project
metric_agg_type: value_count

#query_key: "name.keyword"
doc_type: _type

max_threshold: 32
min_threshold: 19

filter:
- query:
    query_string:
      query: "srctype: geth AND project: hr AND mined AND geth_ip: \"xxx.xxx.xxx.xxx\""

#alert_subject: "No data on dashboard"
# "|" means remove useless characters
alert_text: |
    test attach
alert_text_args: []

alert:
- "email"
email:
- "1803110@narlabes.org.tw"

ElastAlert/config/rule_templates

copy file from github or not copy.
https://github.com/Yelp/elastalert/tree/master/example_rules

run ElastAlert or ElastAlert test

cd /opt/elastalert

a. elastalert

b. elastalert-test-rule ./rules/test_metric.yaml --start 2019-04-10T08:40:00.000

#At /opt/elastalert run command, it auto load config.yaml on this path. You can find "ls config.yaml".

And rules (test_metric.yaml) put on path /opt/elastalert/rules

elastalert.yml and config.json put on path /opt/elastalert-server/config