Q Learning Stochastic Movements about burlap HOT 5 CLOSED

    Visualizer v = GridWorldVisualizer.getVisualizer(gwdg.getMap());
    new EpisodeSequenceVisualizer(v, domain, outputpath);

    gwdg.setProbSucceedTransitionDynamics(transition);
    domain = gwdg.generateDomain();

GridWorldDomain gwdg;
Domain domain;
    GridWorldRewardFunction rf;
    GridWorldTerminalFunction tf;
StateConditionTest goalCondition;
State initialState;
HashableStateFactory hashingFactory;
Environment env;

public AdvancedBehavior(int[][] map, double transition, double movementReward){

    gwdg = new GridWorldDomain(map.length,map[0].length); 
    gwdg.setMap(map);
            gwdg.setProbSucceedTransitionDynamics(transition);
    domain = gwdg.generateDomain();


    rf = new GridWorldRewardFunction(domain, movementReward);
    rf.setReward(map.length-1,map[0].length-1,1.0); // S/B top right corner
    rf.setReward(map.length-1,map[0].length-2,-1.0);
    tf = new GridWorldTerminalFunction();
    tf.markAsTerminalPosition(map.length-1,map[0].length - 1);
    tf.markAsTerminalPosition(map.length-1,map[0].length - 2);

    goalCondition = new TFGoalCondition(tf);

    initialState = GridWorldDomain.getOneAgentNLocationState(domain, 1);
    GridWorldDomain.setAgent(initialState, 0, 0);
    GridWorldDomain.setLocation(initialState, 0, map.length-1,map[0].length - 1);

    hashingFactory = new SimpleHashableStateFactory();

    env = new SimulatedEnvironment(domain, rf, tf, initialState);

}

public void visualize(String outputpath){
    Visualizer v = GridWorldVisualizer.getVisualizer(gwdg.getMap());
    new EpisodeSequenceVisualizer(v, domain, outputpath);
}



public void manualValueFunctionVis(ValueFunction valueFunction, Policy p, String title){

    List<State> allStates = StateReachability.getReachableStates(initialState, 
                                (SADomain)domain, hashingFactory);

    //define color function
    LandmarkColorBlendInterpolation rb = new LandmarkColorBlendInterpolation();
    rb.addNextLandMark(0., Color.RED);
    rb.addNextLandMark(1., Color.BLUE);

    //define a 2D painter of state values, specifying which attributes correspond 
    //to the x and y coordinates of the canvas
    StateValuePainter2D svp = new StateValuePainter2D(rb);
    svp.setXYAttByObjectClass(GridWorldDomain.CLASSAGENT, GridWorldDomain.ATTX,
            GridWorldDomain.CLASSAGENT, GridWorldDomain.ATTY);


    //create our ValueFunctionVisualizer that paints for all states
    //using the ValueFunction source and the state value painter we defined
    ValueFunctionVisualizerGUI gui = new ValueFunctionVisualizerGUI(allStates, svp, valueFunction);
        gui.setTitle(title);
    //define a policy painter that uses arrow glyphs for each of the grid world actions
    PolicyGlyphPainter2D spp = new PolicyGlyphPainter2D();
    spp.setXYAttByObjectClass(GridWorldDomain.CLASSAGENT, GridWorldDomain.ATTX,
            GridWorldDomain.CLASSAGENT, GridWorldDomain.ATTY);
    spp.setActionNameGlyphPainter(GridWorldDomain.ACTIONNORTH, new ArrowActionGlyph(0));
    spp.setActionNameGlyphPainter(GridWorldDomain.ACTIONSOUTH, new ArrowActionGlyph(1));
    spp.setActionNameGlyphPainter(GridWorldDomain.ACTIONEAST, new ArrowActionGlyph(2));
    spp.setActionNameGlyphPainter(GridWorldDomain.ACTIONWEST, new ArrowActionGlyph(3));
    spp.setRenderStyle(PolicyGlyphPainter2D.PolicyGlyphRenderStyle.DISTSCALED);


    //add our policy renderer to it
    gui.setSpp(spp);
    gui.setPolicy(p);

    //set the background color for places where states are not rendered to grey
    gui.setBgColor(Color.BLACK);

    //start it
    gui.initGUI();



}


public void experimentAndPlotter(){

    //different reward function for more interesting results
    ((SimulatedEnvironment)env).setRf(new GoalBasedRF(this.goalCondition, 5.0, -0.1));

    /**
     * Create factories for Q-learning agent and SARSA agent to compare
     */
    LearningAgentFactory qLearningFactory = new LearningAgentFactory() {
        @Override
        public String getAgentName() {
            return "Q-Learning";
        }

        @Override
        public LearningAgent generateAgent() {
            QLearning q = new QLearning(domain, 0.25, hashingFactory, 1.0, .1);
            q.initializeForPlanning(rf,tf,100);
            return (LearningAgent)q;
        }
    };

    LearningAgentFactory sarsaLearningFactory = new LearningAgentFactory() {
        @Override
        public String getAgentName() {
            return "SARSA";
        }

        @Override
        public LearningAgent generateAgent() {
            return new SarsaLam(domain, 0.99, hashingFactory, 0.0, 1.0, 1.);
        }
    };

    LearningAlgorithmExperimenter exp = new LearningAlgorithmExperimenter(env, 10, 100, qLearningFactory);
    exp.setUpPlottingConfiguration(500, 250, 2, 1000,
            TrialMode.MOSTRECENTANDAVERAGE,
            PerformanceMetric.CUMULATIVESTEPSPEREPISODE,
            PerformanceMetric.AVERAGEEPISODEREWARD,
            PerformanceMetric.STEPSPEREPISODE,
            PerformanceMetric.CUMULATIVEREWARDPERSTEP);

    exp.startExperiment();
    exp.writeStepAndEpisodeDataToCSV("expData");

}

 public static int[][] flip_horizontal(int[][] data) {
    //for each row, swap its contents from left to right
    for (int row = 0; row < data.length; row++) {
        for (int col = 0; col < data[0].length / 2; col++) {
            // given a column: i, its pair is column: width() - i - 1
            // e.g. with a width of 10
            // column 0 is paired with column 9
            // column 1 is paired with column 8 etc.
            int temp = data[row][col];
            data[row][col] = data[row][data[0].length - col - 1];
            data[row][data[0].length - col - 1] = temp;
        }
    }
    return data;

}
  public static int[][] getMap(String path, int x, int y){
    try{
            Scanner scanner = new Scanner(new File(path));
            int [][] tall = new int [x][y];
            int i = 0;
            int j = -1;
            while(scanner.hasNextInt()){

            if (i%x == 0)
            {
              j++;
            }
            tall[i % x][j % y] = scanner.nextInt();

            i++;
            }
            tall = flip_horizontal(tall);
            return tall;
            }
            catch (FileNotFoundException e){
                System.out.println("File Not Found");
            }
            catch(Exception e){
                System.out.println("Other Exception " + e.toString());
            }
                return new int[x][y];
        }
public static void main(String[] args) {


           int[][] map = getMap("./src/main/java/maze_larger.txt", 15, 16);

    AdvancedBehavior example = new AdvancedBehavior(map, .1, -.04);
    String outputPath = "output/";


    example.experimentAndPlotter();
    example.visualize(outputPath);
}


}