Elk

https://github.com/appbaseio/dejavu

geth新版的geth.log 會產生豪秒 舊 INFO [04-22|16:29:56] 新 INFO [03-28|13:43:35.004] 差別在.004 看log會發現只有logstash的warning，經同事正確解釋是： >>因為logstash轉換出來的日期格式2019-03-28 13:43:35.004，Elasticsearch不認得，所以對logstash來說是Warning，但對Elasticsearch是Error，造成Log寫不進去~ 所以在logstash.conf上必須在解析date上，補上 “YYYY-MM-dd HH:mm:ss,SSS”, “YYYY-MM-dd HH:mm:ss.SSS” date { match => \[ "gethdate" , "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd HH:mm:ss,SSS", "YYYY-MM-dd HH:mm:ss.SSS"\] target => "gethdate" timezone => "Asia/Taipei" } 這樣就可以正常匯入geth.log了

docker-compose version: '3.3' services: elasticsearch: build: context: elasticsearch/ volumes: #- ./elasticsearch/esdata:/usr/share/elasticsearch/data:rw - alldata:/usr/share/elasticsearch/data:rw #- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro ports: - "9200:9200" - "9300:9300" environment: - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" ulimits: memlock: soft: -1 hard: -1 nofile: soft: 65536 hard: 65536 networks: - fastdev logstash: build: context: logstash/ volumes: - ./logstash/config/logstash.yml:/etc/logstash/logstash.yml:ro - ./logstash/pipeline:/etc/logstash/conf.d:ro ports: - "5000:5000" - "5044:5044" environment: LS_JAVA_OPTS: "-Xmx256m -Xms256m" networks: - fastdev depends_on: - elasticsearch kibana: build: context: kibana/ volumes: - ./kibana/config/:/usr/share/kibana/config:ro ports: - "5601:5601" networks: - fastdev depends_on: - elasticsearch fakelog: build: context: fakelog/ volumes: - ./fakelog/logs:/ethereum-etl/output/:rw networks: - fastdev elastalert: build: context: elastalert/ volumes: - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json:ro - ./elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml:ro - ./elastalert/rules:/opt/elastalert/rules:rw - ./elastalert/rule_templates:/opt/elastalert/rule_templates:ro ports: - "3030:3030" # - "3333:3333" environment: - net=elasticsearch networks: - fastdev volumes: alldata: networks: fastdev: driver: bridge Only elasticsearch/Dockerfile some error happen because debian mirror. Other YAML see before blogs FROM java:8-jre # grab gosu for easy step-down from root ENV GOSU_VERSION 1.7 RUN set -x \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ && export GNUPGHOME="$(mktemp -d)" \ && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ && chmod +x /usr/local/bin/gosu \ && gosu nobody true # https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html # https://packages.elasticsearch.org/GPG-KEY-elasticsearch #RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 #ENV ELASTICSEARCH_MAJOR 1.7 #ENV ELASTICSEARCH_VERSION 1.7.5 #ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/1.7/debian #RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list #RUN set -x \ # && apt-get clean && apt-get update \ # && apt-get install -y --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \ # && rm -rf /var/lib/apt/lists/* RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - RUN echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie.list RUN echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list RUN apt-get -o Acquire::Check-Valid-Until=false update && apt-get install elasticsearch ENV PATH /usr/share/elasticsearch/bin:$PATH WORKDIR /usr/share/elasticsearch RUN set -ex \ && for path in \ ./data \ ./logs \ ./config \ ./config/scripts \ ; do \ mkdir -p "$path"; \ chown -R elasticsearch:elasticsearch "$path"; \ done COPY config ./config #VOLUME /usr/share/elasticsearch/data COPY docker-entrypoint.sh / RUN chmod +x /docker-entrypoint.sh EXPOSE 9200 9300 ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["elasticsearch"] ElastAlert/Dockerfile FROM bitsensor/elastalert:latest ElastAlert/config/config.json { "appName": "elastalert-server", "port": 3030, "wsport": 3333, "elastalertPath": "/opt/elastalert", "verbose": false, "es_debug": false, "debug": false, "rulesPath": { "relative": true, "path": "/rules" }, "templatesPath": { "relative": true, "path": "/rule_templates" }, "es_host": "elasticsearch", "es_port": 9200, "writeback_index": "elastalert_status" } ElastAlert/config/elastalert.yaml es_host: elasticsearch es_port: 9200 rules_folder: rules # How often ElastAlert will query elasticsearch # The unit can be anything from weeks to seconds run_every: seconds: 5 #minutes: 1 # ElastAlert will buffer results from the most recent # period of time, in case some log sources are not in real time buffer_time: minutes: 15 # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed alert_time_limit: minutes: 2 #days: 2 writeback_index: elastalert_status #smtp_host: "" ElastAlert/config/rules/test_metric.yaml name: Example rule # How often ElastAlert will query elasticsearch # The unit can be anything from weeks to seconds run_every: seconds: 5 #minutes: 1 # ElastAlert will buffer results from the most recent # period of time, in case some log sources are not in real time buffer_time: minutes: 15 # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed alert_time_limit: minutes: 2 #days: 2 type: metric_aggregation index: filebeat-* metric_agg_key: project metric_agg_type: value_count #query_key: "name.keyword" doc_type: _type max_threshold: 32 min_threshold: 19 filter: - query: query_string: query: "srctype: geth AND project: hr AND mined AND geth_ip: \"xxx.xxx.xxx.xxx\"" #alert_subject: "No data on dashboard" # "|" means remove useless characters alert_text: | test attach alert_text_args: [] alert: - "email" email: - "1803110@narlabes.org.tw" ElastAlert/config/rule_templates copy file from github or not copy. https://github.com/Yelp/elastalert/tree/master/example_rules ...

ethereum-etl export data every time write over file. Filebeat always reload file then ELK receive repeat records… So .env STARTBLOCK=01205866 ENDBLOCK=01205888 startetl.sh IP_PORT is go-ethereum-node1 outside ip & port. here is docker-machine source get file that path need to be careful #!/bin/bash IP_PORT=192.168.99.100:18545 ETH_METHOD=eth_blockNumber BLOCKNUMBER_JSON_HEX=$(curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"'$ETH_METHOD'","params":[],"id":1}' $IP_PORT | jq '.result' | tr -d '"') BLOCKNUMBER_DEX=$(printf "%08d\n" $BLOCKNUMBER_JSON_HEX) printf "\n===== Now Geth BlockerNumber =====\n" printf "HEX: %s\n" $BLOCKNUMBER_JSON_HEX printf "DEC: %s\n" $BLOCKNUMBER_DEX source .env #source ~/ethereum_etl/.env printf "\n===== .env BLOCK VARS =====\n" printf "STARTBLOCK: %s\n" $STARTBLOCK printf "ENDBLOCK: %s\n" $ENDBLOCK if [[ $BLOCKNUMBER_DEX =~ ^?[0-9]+$ ]] || [ -z $BLOCKNUMBER_DEX ]; then printf "NOT Number Geth BLOCKNUMBER, check Geth\n" exit fi if [[ $STARTBLOCK =~ ^?[0-9]+$ ]] || [ -z $STARTBLOCK ]; then printf "NOT Number .env STARTBLOCK, check .env\n" exit fi if [[ $ENDBLOCK =~ ^?[0-9]+$ ]] || [ -z $ENDBLOCK ]; then printf "NOT Number .env ENDBLOCK, check .env\n" exit fi #printf "\n%s\n" $BLOCKNUMBER_DEX #printf "%s" $STARTBLOCK #printf "\n%s\n" $((10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK)) if (( (( 10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK -1 )) > 0 )); then printf "\n===== Change .env =====\n" printf "startblock: %s => %s\n" $STARTBLOCK $((10#$ENDBLOCK+1)) printf "endblock: %s => %s\n" $ENDBLOCK $BLOCKNUMBER_DEX STARTBLOCK=$(printf "%08d\n" $((10#$ENDBLOCK+1)) ) echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env echo ENDBLOCK=$BLOCKNUMBER_DEX >> .env #echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env #echo ENDBLOCK=$BLOCKNUMBER_DEX >> ~ethereum_etl/.env fi docker-compose up --build ethereum_etl docker-compose.yml version: '3.3' services: go-ethereum-node1: build: context: go-ethereum-node1/ volumes: #- ./go-ethereum/keystore:/root/.ethereum/devchain/keystore:rw - ./go-ethereum-node1/genesis/poa_for_dev.json:/root/genesis/poa_for_dev.json:ro - alldata:/root/.ethereum/devchain - /etc/localtime:/etc/localtime:ro #- ./go-ethereum/log:/root/log:rw entrypoint: /root/start.sh ports: - "18545:8545" - "30313:30303" - "30313:30303/udp" networks: - etl ethereum_etl: build: context: ethereum-etl/ env_file: .env volumes: - alldata:/ethereum-etl/output:rw #- /root/go/src/github.com/ethereum/go-ethereum/build/bin/data:/ethereum-etl/ipc #restart: unless-stopped networks: - etl volumes: alldata: networks: etl: driver: bridge ethereum_etl DOCKERFILE FROM python:3.6-alpine MAINTAINER Eric Lim ENV PROJECT_DIR=ethereum-etl RUN apk add unzip RUN wget https://github.com/blockchain-etl/ethereum-etl/archive/develop.zip \ && unzip develop.zip && rm develop.zip RUN mv ethereum-etl-develop /$PROJECT_DIR WORKDIR /$PROJECT_DIR RUN apk add --no-cache gcc musl-dev #for C libraries: RUN pip install --upgrade pip && pip install -e /$PROJECT_DIR/ #CMD ["export_all", "-s", "01990000", "-e", "99999999", "-p", "http://xxx.xxx.xxx.xxx:8545", "-o", "output"] #CMD ["sh","-c", "echo startblock=$STARTBLOCK endblock=$ENDBLOCK"] CMD ["sh","-c","python ethereumetl export_all -s $STARTBLOCK -e $ENDBLOCK -p http://xxx.xxx.xxx.xxx:8545 -o output"] go-ethereum-node1 see https://sueboy.blogspot.com/search?q=poa ...

https://github.com/sueboy19/oldelk