yuxuanh / logstash-input-sftp Goto Github PK

input {
    sftp {
        username => "test"
        password => "123"
        remote_host => "10.10.10.3"
        port => 22
        remote_path => "/home/test/123.txt"
        local_path => "/inputFiles/sftp_download/123.txt"
        schedule => "* * * * *"
        add_field => {
            "channelId" => 1
        }
    }
}
output {
  stdout {
    codec => rubydebug
  }
}

[2019-10-12T07:28:02,432][INFO ][logstash.inputs.sftp     ] /inputFiles/sftp_download/123.txt has processed, now waiting * * * * *, then it will download and process again
{
       "message" => "logstash_input_file001001001",
      "@version" => "1",
    "@timestamp" => 2019-10-12T07:28:02.430Z
}
{
       "message" => "logstash_input_file001001002",
      "@version" => "1",
    "@timestamp" => 2019-10-12T07:28:02.430Z
}

=> [6/7] RUN zip -r logstash-input-sftp.zip logstash                                                                                                                                               0.6s 
 => ERROR [7/7] RUN bin/logstash-plugin install logstash-input-sftp.zip                                                                                                                             6.4s 
------
 > [7/7] RUN bin/logstash-plugin install logstash-input-sftp.zip:
#0 0.563 Using bundled JDK: /usr/share/logstash/jdk
#0 6.316 ERROR: Something went wrong when installing logstash-input-sftp.zip, message: certificate verify failed

FROM docker.elastic.co/logstash/logstash:7.16.2

COPY ./logstash-input-sftp/logstash-input-sftp/logstash-input-sftp.gemspec ./
# RUN apt-get install rubygems
user root 

# RUN yum install -y ruby ruby-devel rubygems
RUN yum install -y wget zip
RUN wget --no-check-certificate https://rubygems.org/downloads/net-ssh-5.0.2.gem \
 && wget --no-check-certificate https://rubygems.org/downloads/net-sftp-2.1.2.gem \
&& mkdir logstash 
RUN mv *.gem logstash/
RUN zip -r logstash-input-sftp.zip logstash 
RUN bin/logstash-plugin install logstash-input-sftp.zip

user logstash

# RUN gem build logstash-input-sftp.gemspec

I'm using a VPN because I'm using a company laptop, when I run the command RUN bin/logstash-plugin install logstash-input-sftp.zip in the DockerFile I get this error

=> [6/7] RUN zip -r logstash-input-sftp.zip logstash                                                                                                                                               0.6s 
 => ERROR [7/7] RUN bin/logstash-plugin install logstash-input-sftp.zip                                                                                                                             6.4s 
------
 > [7/7] RUN bin/logstash-plugin install logstash-input-sftp.zip:
#0 0.563 Using bundled JDK: /usr/share/logstash/jdk
#0 6.316 ERROR: Something went wrong when installing logstash-input-sftp.zip, message: certificate verify failed
My Dockerfile is:

FROM docker.elastic.co/logstash/logstash:7.16.2

COPY ./logstash-input-sftp/logstash-input-sftp/logstash-input-sftp.gemspec ./
# RUN apt-get install rubygems
user root 

# RUN yum install -y ruby ruby-devel rubygems
RUN yum install -y wget zip
RUN wget --no-check-certificate https://rubygems.org/downloads/net-ssh-5.0.2.gem \
 && wget --no-check-certificate https://rubygems.org/downloads/net-sftp-2.1.2.gem \
&& mkdir logstash 
RUN mv *.gem logstash/
RUN zip -r logstash-input-sftp.zip logstash 
RUN bin/logstash-plugin install logstash-input-sftp.zip

user logstash

# RUN gem build logstash-input-sftp.gemspec

My docker-compose.yml is:

version: '3.8'
services:

  logstash:
    # image: sebp/elk
    image: logstash-sftp:7.16.2
    environment:
      - "ES_JAVA_OPTS=-Xmx256m -Xms256m"
      - "ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST}"
      - "ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME}"
      - "ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD}"
      - "CONFIG_RELOAD_AUTOMATIC=true"
      - "PIPELINE_WORKERS=1"
      # - "PIPELINE_BATCH_DELAY=1"
      - "xpack_monitoring_enabled=false"
    # ports:
    #   - 5000:5000
    volumes:
      # - ./logstash/config:/usr/share/logstash/config
      - ./logstash/data:/usr/share/logstash/data
      - ./logstash/config/pipeline:/usr/share/logstash/pipeline
      # - ./logstash/logs:/usr/share/logstash/logs
    networks:
      - elastic

volumes:
  elasticsearch_data:

networks:
  elastic: