% APRENDIZAJE AUTOMÁTICO (G. Pajares y J. M. de la Cruz, Eds.)
% Editorial: RA-MA
% Capítulo 12: Aprendizaje por Refuerzo
% Autores: Jose Antonio Martin H y Matilde Santos
function  Cart_PoleDemo( maxepisodes )
%Cart_PoleDemo, the main function of the demo
%maxepisodes: maximum number of episodes to run the demo

% Cart Pole control demo with SARSA 
% Programmed in Matlab 
% by:
%  Jose Antonio Martin H. <jamartinh@fdi.ucm.es>

set(gcf,'name','Reinforcement Learning for the Cart-Pole problem');
%set(gcf,'Color','w')
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
grid off						% turns on grid
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
set(gcf,'BackingStore','off') 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                         REINFORCEMENT LEARNING LOOP
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
drawnow;


global alpha
global gamma
global epsilon
global grafica
global Q
global statelist
global kind_reward
global f2
global nstates
global nactions
global continue_test
global reward_equilibrium
global reward_fall
global kind_exp
global i
global xpoints
global ypoints

maxsteps    = 1000;              % maximum number of steps per episode
statelist   = BuildStateList();  % the list of states
actionlist  = BuildActionList(); % the list of actions

nstates     = size(statelist,1);
nactions    = size(actionlist,1);

if isempty(Q)
Q           = BuildQTable( nstates,nactions );  % the Qtable
end

if isempty(alpha)
alpha       = 0.3;   % learning rate
end
if isempty(gamma)
gamma       = 1.0;   % discount factor
end

if isempty(epsilon)
epsilon     = 0.1;  % probability of a random action selection
end

if isempty(reward_equilibrium)
reward_equilibrium =  0.0;
reward_fall        = -1.0;
end

if isempty(kind_exp)
    kind_exp=2;
end

  

if isempty(grafica)
grafica     = false; % indicates if display the graphical interface
end


continue_test = true;

if isempty(xpoints)
xpoints     = [];
ypoints     = [];
end
if isempty(i)
i = 1;
end
while i<=maxepisodes && continue_test
    
    [total_reward,steps ] = Episode( maxsteps,statelist,actionlist ); 
    if continue_test==false        
        return
    end
    disp(['Espisode: ',int2str(i),'  Steps:',int2str(steps),'  Reward:',num2str(total_reward),' epsilon: ',num2str(epsilon)])
    
    epsilon = epsilon * 0.99;
    
    xpoints(i)=i-1;
    ypoints(i)=steps;    
    
    if isempty(f2)
        f2 = subplot(2,1,2);    
    end
    plot(f2,xpoints,ypoints) 
    axis(f2,[0 i 0 1100])
    title(f2,['Episode: ',int2str(i),' epsilon: ',num2str(epsilon)])  
    xlabel(f2,'Episodes')
    ylabel(f2,'Steps')        
    drawnow
    
    if (i>2000)
        grafica=true;
    end
    
    i=i+1;
end






