ethereum etl ELK docker docker-compose .env startblock endblock

ethereum-etl export data every time write over file. Filebeat always reload file then ELK receive repeat records...

So

.env

STARTBLOCK=01205866
ENDBLOCK=01205888

startetl.sh

IP_PORT is go-ethereum-node1 outside ip & port. here is docker-machine
source get file that path need to be careful

#!/bin/bash

IP_PORT=192.168.99.100:18545
ETH_METHOD=eth_blockNumber
BLOCKNUMBER_JSON_HEX=$(curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"'$ETH_METHOD'","params":[],"id":1}' $IP_PORT | jq '.result' | tr -d '"')
BLOCKNUMBER_DEX=$(printf "%08d\n" $BLOCKNUMBER_JSON_HEX)

printf "\n===== Now Geth BlockerNumber =====\n"
printf "HEX: %s\n" $BLOCKNUMBER_JSON_HEX
printf "DEC: %s\n" $BLOCKNUMBER_DEX

source .env
#source ~/ethereum_etl/.env

printf "\n===== .env BLOCK VARS =====\n"
printf "STARTBLOCK: %s\n" $STARTBLOCK
printf "ENDBLOCK: %s\n" $ENDBLOCK

if [[ $BLOCKNUMBER_DEX =~ ^?[0-9]+$ ]] || [ -z $BLOCKNUMBER_DEX ]; then
  printf "NOT Number Geth BLOCKNUMBER, check Geth\n"
  exit
fi

if [[ $STARTBLOCK =~ ^?[0-9]+$ ]] || [ -z $STARTBLOCK ]; then
  printf "NOT Number .env STARTBLOCK, check .env\n"
  exit
fi

if [[ $ENDBLOCK =~ ^?[0-9]+$ ]] || [ -z $ENDBLOCK ]; then
  printf "NOT Number .env ENDBLOCK, check .env\n"
  exit
fi

#printf "\n%s\n" $BLOCKNUMBER_DEX
#printf "%s" $STARTBLOCK
#printf "\n%s\n" $((10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK))

if (( (( 10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK -1 )) > 0 )); then
  printf "\n===== Change .env =====\n"
  printf "startblock: %s => %s\n" $STARTBLOCK $((10#$ENDBLOCK+1))
  printf "endblock: %s => %s\n" $ENDBLOCK $BLOCKNUMBER_DEX  

  STARTBLOCK=$(printf "%08d\n" $((10#$ENDBLOCK+1)) )
  echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env
  echo ENDBLOCK=$BLOCKNUMBER_DEX >> .env
  #echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env
  #echo ENDBLOCK=$BLOCKNUMBER_DEX >> ~ethereum_etl/.env
fi

docker-compose up --build ethereum_etl

docker-compose.yml

version: '3.3'

services:
  go-ethereum-node1:
    build:
      context: go-ethereum-node1/
    volumes:
      #- ./go-ethereum/keystore:/root/.ethereum/devchain/keystore:rw
      - ./go-ethereum-node1/genesis/poa_for_dev.json:/root/genesis/poa_for_dev.json:ro
      - alldata:/root/.ethereum/devchain
      - /etc/localtime:/etc/localtime:ro
      #- ./go-ethereum/log:/root/log:rw
    entrypoint: /root/start.sh
    ports:
      - "18545:8545"
      - "30313:30303"
      - "30313:30303/udp"
    networks:
      - etl

  ethereum_etl:
    build:
      context: ethereum-etl/
    env_file: .env
    volumes:
      - alldata:/ethereum-etl/output:rw
      #- /root/go/src/github.com/ethereum/go-ethereum/build/bin/data:/ethereum-etl/ipc
    #restart: unless-stopped
    networks:
      - etl

volumes:  
  alldata: 

networks:
  etl:
    driver: bridge

ethereum_etl DOCKERFILE

FROM python:3.6-alpine
MAINTAINER Eric Lim 
ENV PROJECT_DIR=ethereum-etl

RUN apk add unzip
RUN wget https://github.com/blockchain-etl/ethereum-etl/archive/develop.zip \
&& unzip develop.zip && rm develop.zip

RUN mv ethereum-etl-develop /$PROJECT_DIR

WORKDIR /$PROJECT_DIR
RUN apk add --no-cache gcc musl-dev  #for C libraries:  
RUN pip install --upgrade pip && pip install -e /$PROJECT_DIR/

#CMD ["export_all", "-s", "01990000", "-e", "99999999", "-p", "http://xxx.xxx.xxx.xxx:8545", "-o", "output"]
#CMD ["sh","-c", "echo startblock=$STARTBLOCK  endblock=$ENDBLOCK"]

CMD ["sh","-c","python ethereumetl export_all -s $STARTBLOCK -e $ENDBLOCK  -p http://xxx.xxx.xxx.xxx:8545 -o output"]

go-ethereum-node1

see https://sueboy.blogspot.com/search?q=poa