\begin{thebibliography}{13}
\expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi
\expandafter\ifx\csname url\endcsname\relax
  \def\url#1{{\tt #1}}\fi

\bibitem[Azuma(1967)]{azuma}
K.~Azuma.
\newblock Weighted sums of certain dependent random variables.
\newblock {\em Tohoku Mathematical Journal}, 68:\penalty0 357--367, 1967.

\bibitem[Beleznay et~al.(1999)Beleznay, Grobler, and Szepesvari]{bgs99}
F.~Beleznay, T.~Grobler, and C.~Szepesvari.
\newblock Comparing value-function estimation algorithms in undiscounted
  problems.
\newblock Technical Report TR-99-02, Mindmaker Ltd, 1999.

\bibitem[Bertsekas and Tsitsiklis(1996)]{bt96}
Dimitri~P. Bertsekas and John~N. Tsitsiklis.
\newblock {\em Neuro-Dynamic Programming}.
\newblock Athena Scientific, Belmont, MA, 1996.

\bibitem[Borkar and Meyn(2000)]{ode}
V.S. Borkar and S.P. Meyn.
\newblock The {O.D.E} method for convergence of stochstic approximation and
  reinforcement learning.
\newblock {\em Siam J. Control}, 38 (2):\penalty0 447--69, 2000.

\bibitem[Jaakkola et~al.(1994)Jaakkola, Jordan, and Singh]{jakcon}
Tommi Jaakkola, Michael~I. Jordan, and Satinder~P. Singh.
\newblock On the convergence of stochastic iterative dynamic programming
  algorithms.
\newblock {\em Neural Computation, 6}, 1994.

\bibitem[Kearns and Singh(1999)]{ks98}
Michael Kearns and Satinder~P. Singh.
\newblock Finite-sample convergence rates for {Q}-learning and indirect
  algorithms.
\newblock In M.J. Kearns, S.A. Solla, and D.A. Cohn, editors, {\em Advances in
  Neural Information Processing Systems 11}, pages 996--1002, 1999.

\bibitem[Littman and Szepesv\'{a}ri(1996)]{lit96}
Michael~L. Littman and Gaba Szepesv\'{a}ri.
\newblock A generalized reinforcement-learning model: {C}onvergence and
  applications.
\newblock In L.~Saitta, editor, {\em Proceedings of the 13th International
  Conference on Machine Learning ({ICML}-96)}, pages 310--318, Bari, Italy,
  1996. Morgan Kaufmann.
\newblock URL \url{citeseer.nj.nec.com/littman96generalized.html}.

\bibitem[Puterman.(1994)]{Puterman}
Martin~L. Puterman.
\newblock {\em Markov Decision Processes|Discrete Stochastic Dynamic
  Programming}.
\newblock John Wiley and Sons, Inc., New York, NY, 1994.

\bibitem[Sutton and Barto(1998)]{sb98}
R.~Sutton and A.~Barto.
\newblock {\em Reinforcement Learning}.
\newblock MIT Press., Cambridge, MA., 1998.

\bibitem[Szepesvari(1998)]{s97}
C.~Szepesvari.
\newblock The asymptotic convergence-rate of {Q}-learning.
\newblock In M.I. Jordan, M.J. Kearns, and S.A. Solla, editors, {\em Advances
  in Neural Information Processing Systems 10}, pages 1064--1070, 1998.

\bibitem[Tsitsiklis(1994)]{tsi94}
J.~Tsitsiklis.
\newblock Asynchronous stochastic approximation and {Q}-learning, 1994.

\bibitem[Watkins and Dyan(1992)]{wat92}
C.~Watkins and P.~Dyan.
\newblock {Q}-learning.
\newblock {\em Machine Learning}, 8(3/4):\penalty0 279--292, 1992.

\bibitem[Watkins(1989)]{wat89}
C.~J. C.~H. Watkins.
\newblock {\em Learning from Delayed Rewards}.
\newblock PhD thesis, Cambridge, England, 1989.

\end{thebibliography}
