% Do a Good-Turing smoothing for given data

[F FF]=textread('GTtable.txt',' %d %d');

% Plot
hold off
loglog(F,FF,'.')
axis([1 1e4 0.5 1e7])
xlabel('Frekvenssi r');
ylabel('Kuinka monenlaisia trigrammeja esiintyy r kertaa');
title('Raaka data');
print -depsc GTraaka.eps

% There is something weird in the higher counts ?
% Lets smooth...
il=F-[0; F(1:length(F)-1)];
FFsmooth=FF./il;
hold off
loglog(F,FFsmooth)

% Linear regression:
regm=[ones(length(F),1) log(F)];
coeffi=regm\log(FFsmooth);
hold on

FFreg=exp([ones(size(F))  log(F) ]*coeffi);
loglog(F,FFreg,'m');
xlabel('Frekvenssi r');
ylabel('Kuinka monenlaisia trigrammeja esiintyy r kertaa');
title('Tasoiteltu data ja regressiosuora');
print -depsc GTsmooth.eps

% Regressed counts:
rstar=(F+1).*exp([ones(size(F))  log(F+1) ]*coeffi - [ones(size(F)) log(F) ]*coeffi);

% Probability mass given to unseen events:
p0=FF(1)./sum(F.*FF);

%rstar2=[(F(1:30)+1).*FF(2:31)./FF(1:30) ; rstar(31:length(rstar)) ];
rstar2=[(F(1:15)+1).*FF(2:16)./FF(1:15) ; rstar(16:length(rstar)) ];
probs=(1-p0)*[rstar2/sum(FF.*rstar2)];
probs=[p0 ;probs]

% Check that all sums to 1
fprintf(1,'The following number should be very close to 1.0: %f\n',...
	[ 1 FF']*probs);
