% APRENDIZAJE AUTOMÁTICO (G. Pajares y J. M. de la Cruz, Eds.)
% Editorial: RA-MA
% Capítulo 12: Aprendizaje por Refuerzo
% Autores: Jose Antonio Martin H y Matilde Santos

function [pa  a] = softmax_selection( Q , s, epsilon )
% e_greedy_selection selects an action using Epsilon-greedy strategy
% Q: the Qtable
% s: the current state
actions = size(Q,2);
if (epsilon<=0)
     a  = GetBestAction(Q,s); 
     pa = 1.0/actions;
     return
end

%Calculate Gibss distribution
ea     = exp( Q(s,:)/epsilon );
sum_ea = sum( ea );

% check if out of numeric precition
if (sum_ea==0 || isnan(sum_ea) || any(isnan(ea))) 
    a = GetBestAction(Q,s); 
    pa = 1.0/actions;
    return
end


%Calculate probabilities
values = ea /(sum_ea);

if (any(isnan(values)))
    a = GetBestAction(Q,s); 
    pa = 1.0/actions;
    return
end

    
%values
%suma = sum(values)

[v a] = max(Q(s,:).*-log2(values) );
pa = values(a);
return
Y   = cumsum(values);
z   = rand(); 

for i=1:actions
    if z <= Y(i)        
            a  = i;
            pa = values(a);
            return
    end     
end

a  = actions;
pa = values(a);



 
