Code Monkey home page Code Monkey logo

spark's Introduction

Spark

package driver;

import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner;

import mapper.MoviesMapper; import reducer.MoviesReducer;

public class MoviesDriver extends Configured implements Tool {

public static void main(String[] args) {
	if (args.length < 2) {
		System.out.println("java usage " + MoviesDriver.class.getName()
				+ "/Path/To/HDFS/input/File/  /Output/HDFS/Path/Directory/");
		return;
	}

	Configuration conf = new Configuration(Boolean.TRUE);

	try {
		int i = ToolRunner.run(conf, new MoviesDriver(), args);
		System.out.println("i---->" + i);
		if (i == 0) {
			System.out.println("Success!!");
		} else {
			System.out.println("Failure!!");
		}
	} catch (Exception e) {
		e.printStackTrace();
		System.out.println("Failed in Stack Trace");
	}
}

public int run(String[] args) throws Exception {

	// Step -1 : Get the configuration
	
	Configuration conf = super.getConf();

	// Step -2 : Create a Job Instance

	Job movies = Job.getInstance(conf, MoviesDriver.class.getName());

	// Step -3 : Setting the classpath using the jar file 
	
	movies.setJarByClass(MoviesDriver.class);
	
	// Step -4 : Setting the Input
	
	TextInputFormat.addInputPath(movies, new Path(args[0]));
	movies.setInputFormatClass(TextInputFormat.class);
	
	// Step -5 : Setting the output
	
	TextOutputFormat.setOutputPath(movies, new Path(args[1]));
	movies.setOutputFormatClass(TextOutputFormat.class);
	
	// Step -6 : Setting the Mapper
	
	movies.setMapperClass(MoviesMapper.class);
	
	// Step -7 : Setting Reducer Class
	
	movies.setReducerClass(MoviesReducer.class);
	
	// Step InBetween : Set number of reduce task
	
	movies.setNumReduceTasks(4);
	
	// Step -8 : Setting the Mapper output key and Value 
	
	movies.setMapOutputKeyClass(IntWritable.class);
	movies.setMapOutputValueClass(IntWritable.class);
	
	//Step -9 : Set the Reducer Output Key and Value Class
	
	movies.setOutputKeyClass(IntWritable.class);
	movies.setOutputValueClass(IntWritable.class);
	
	
	
	movies.waitForCompletion(Boolean.TRUE);
	return 0;
}

}

spark's People

Contributors

manerm27 avatar

Watchers

James Cloos avatar

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    ๐Ÿ–– Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. ๐Ÿ“Š๐Ÿ“ˆ๐ŸŽ‰

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google โค๏ธ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.