keedio / flume-taildirectory-source Goto Github PK

           flume custom source for ingesting local files

           flume custom multi-source for ingestion files from different sources.

for (FileSet fileSet: fileSetMap.values()){

                    [.......]
                    if (currentTime - lastAppendTime > TimeUnit.MINUTES.toMillis(timeToUnlockFile)){
                        logger.info("File: " + fileSet.getFilePath() + 
                                " not modified after " + timeToUnlockFile + " minutes" +
                                " removing from monitoring list");
                        fileSetMap.get(fileKey).clear();
                        fileSetMap.get(fileKey).close();
                        fileSetMap.remove(fileKey);
                        filePathsAndKeys.remove(fileSet.getFilePath().toString());
                    }
                }