% APRENDIZAJE AUTOMÁTICO (G. Pajares y J. M. de la Cruz, Eds.)
% Editorial: RA-MA
% Capítulo 12: Aprendizaje por Refuerzo
% Autores: Jose Antonio Martin H y Matilde Santos
function [ total_reward,steps ] = Episode( maxsteps,statelist,actionlist )


global grafica
global continue_test
global kind_exp
global Q
global alpha
global gamma
global epsilon

% state variables x,x_dot,theta,theta_dot
x            = [0 0 0 0.01];
steps        = 0;
total_reward = 0;


% convert the continous state variables to an index of the statelist
s   = DiscretizeState(x,statelist);


if kind_exp==1
% selects an action using the epsilon greedy selection strategy
    a   = e_greedy_selection(Q,s,epsilon);
else    
    [p, a] = softmax_selection( Q , s, epsilon );
end



for i=1:maxsteps    
        
    % convert the index of the action into an action value
    action = actionlist(a);    
    
    %do the selected action and get the next car state    
    xp  = DoAction( action , x );    
    
    % observe the reward at state xp and the final state flag
    [r,f]   = GetReward(xp);
    total_reward = total_reward + r;
    
    % convert the continous state variables in [xp] to an index of the statelist    
    sp  = DiscretizeState(xp,statelist);
    
    
    % select action prime
    if kind_exp==1
        ap = e_greedy_selection(Q,sp,epsilon);   
    else
        [p, ap] = softmax_selection( Q , sp, epsilon );
    end

    
    % Update the Qtable, that is,  learn from the experience
    Q = UpdateSARSA( s, a, r, sp, ap, Q , alpha, gamma);
       
    
    %update the current variables
    s = sp;
    a = ap;
    x = xp;
    
    
   %increment the step counter.
   steps=steps+1;
    
   grafic=grafica;
    % Plot of the mountain car problem
    if (grafic==true && continue_test==true)        
       plot_Cart_Pole(x,action,steps);    
    end
    
    % if the car reachs the goal breaks the episode
    if (f==true|| continue_test==false)
        break
    end
    
end


