\begin{thebibliography}{58}
\expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi
\expandafter\ifx\csname url\endcsname\relax
  \def\url#1{{\tt #1}}\fi

\bibitem[Ackley et~al.(1985)Ackley, Hinton, and Sejnowski]{AckHinSej85}
D.~H. Ackley, G.~E. Hinton, and T.~J. Sejnowski.
\newblock A learning algorithm for {Boltzmann} machines.
\newblock {\em Cognitive Science}, 9:\penalty0 147--169, 1985.

\bibitem[Baird and Klopf(1993)]{BaiKlo93}
L.~Baird and H.~Klopf.
\newblock Reinforcement learning with high-dimensional continuous actions.
\newblock {Technical Report WL-TR-93-1147}, Wright Laboratory, Wright-Patterson
  Air Force Base, 1993.

\bibitem[Baird and Moore(1999)]{BaiMoo99}
L.~Baird and A.~Moore.
\newblock Gradient descent for general reinforcement learning.
\newblock In M.~S. Kearns, S.~A. Solla, and D.~A. Cohn, editors, {\em Advances
  in Neural Information Processing Systems}, volume~11. The {MIT} Press,
  Cambridge, 1999.

\bibitem[Barto et~al.(1983)Barto, Sutton, and Anderson]{BarEtAl83}
A.~G. Barto, R.~S. Sutton, and C.~W. Anderson.
\newblock Neuronlike adaptive elements that can solve difficult learning
  control problems.
\newblock {\em IEEE Transactions on Systems, Man and Cybernetics}, 13:\penalty0
  835--846, 1983.

\bibitem[Bellman(1957{\natexlab{a}})]{Bell57}
R.~E. Bellman.
\newblock {\em Dynamic Programming}.
\newblock Princeton University Press, Princeton, NJ, 1957{\natexlab{a}}.

\bibitem[Bellman(1957{\natexlab{b}})]{Bell57b}
R.~E. Bellman.
\newblock A {M}arkov decision process.
\newblock {\em Journal of Mathematical Mechanics}, 6:\penalty0 679--684,
  1957{\natexlab{b}}.

\bibitem[Bertsekas and Tsitsiklis(1996)]{BerTsi96}
D.~P. Bertsekas and J.~N. Tsitsiklis.
\newblock {\em Neuro-Dynamic Programming}.
\newblock Athena Scientific, Belmont, MA, 1996.

\bibitem[Boutilier et~al.(2001)Boutilier, Bacchus, and Brafman]{BouEtAl01}
C.~Boutilier, F.~Bacchus, and R.~I. Brafman.
\newblock Ucp-networks: A directed graphical representation of conditional
  utilities.
\newblock In {\em Proceedings of the Seventeenth Annual Conference on
  Uncertainty in Artificial Intelligence (UAI-01)}, pages 56--64, 2001.

\bibitem[Boutilier et~al.(1999)Boutilier, Brafman, Hoos, and Poole]{BouEtAl99b}
C.~Boutilier, R.~I. Brafman, H.~H. Hoos, and D.~Poole.
\newblock Reasoning with conditional ceteris paribus preference statements.
\newblock In {\em Proceedings of the Sixth International Conference on
  Principles of Knowledge Representation and Reasoning}, pages 71--80, 1999.

\bibitem[Boutilier et~al.(2000)Boutilier, Dearden, and Goldszmidt]{BouEtAl00}
C.~Boutilier, R.~Dearden, and M.~Goldszmidt.
\newblock Stochastic dynamic programming with factored representations.
\newblock {\em Articial Intelligence}, 121:\penalty0 49--107, 2000.

\bibitem[Boutilier and Poole(1996)]{BouPoo96}
C.~Boutilier and D.~Poole.
\newblock Computing optimal policies for partially observable decision
  processes using compact representations.
\newblock In {\em Proc. AAAI-96}, 1996.

\bibitem[Brown and Hinton(2001)]{BroHin01}
A.~D. Brown and G.~E. Hinton.
\newblock Products of hidden {M}arkov models.
\newblock In T.~S. Jaakkola and T.~Richardson, editors, {\em Proceedings of
  Artificial Intelligence and Statistics 2001}, pages 3--11, 2001.

\bibitem[Cooper(1990)]{Coo90}
G.~F. Cooper.
\newblock The computational complexity of probabilistic inference using
  {B}ayesian belief networks.
\newblock {\em Artificial Intelligence}, 42:\penalty0 393--405, 1990.

\bibitem[Cowell et~al.(1999)Cowell, Dawid, Lauritzen, and
  Spiegelhalter]{CowEtAl99}
R.~G. Cowell, A.~P. Dawid, S.~L. Lauritzen, and D.~J. Spiegelhalter.
\newblock {\em Probabilistic Networks and Expert Systems}.
\newblock Springer-Verlag, 1999.

\bibitem[Dean et~al.(1998)Dean, Givan, and Kim]{DeaEtAl98}
T.~Dean, R.~Givan, and K.~Kim.
\newblock Solving stochastic planning problems with large state and action
  spaces.
\newblock In {\em Proc. Fourth International Conference on Artificial
  Intelligence Planning Systems}, 1998.

\bibitem[Dean and Kanazawa(1989)]{DeaKan89}
T.~Dean and K.~Kanazawa.
\newblock A model for reasoning about persistence and causation.
\newblock {\em Computational Intelligence}, 5, 1989.

\bibitem[Dietterich(2000)]{Die00}
T.~Dietterich.
\newblock Hierarchical reinforcement learning with the {MAXQ} value function
  decomposition.
\newblock {\em Journal of Artificial Intelligence Research}, 13:\penalty0
  227--303, 2000.

\bibitem[Duane et~al.(1987)Duane, Kennedy, Pendleton, and Roweth]{DuaEtAl87}
S.~Duane, A.~D. Kennedy, B.~J. Pendleton, and D.~Roweth.
\newblock Hybrid monte carlo.
\newblock {\em Physics Letters B}, 195:\penalty0 216--222, 1987.

\bibitem[Freund and Haussler(1992)]{FreHau92}
Y.~Freund and D.~Haussler.
\newblock Unsupervised learning of distributions on binary vectors using two
  layer networks.
\newblock In John~E. Moody, Steven~J. Hanson, and Richard~P. Lippmann, editors,
  {\em Advances in Neural Information Processing Systems}, volume~4. Morgan
  Kaufmann , San Mateo, 1992.

\bibitem[Geman and Geman(1984)]{GemGem84}
S.~Geman and D.~Geman.
\newblock Stochastic relaxation, {G}ibbs distributions, and the {{Bayesian}}
  restoration of images.
\newblock {\em IEEE Transactions on Pattern Analysis and Machine Intelligence},
  6:\penalty0 721--741, 1984.

\bibitem[Guestrin et~al.(2002)Guestrin, Koller, and Parr]{GueEtAl02}
C.~Guestrin, D.~Koller, and R.~Parr.
\newblock Multiagent planning with factored {MDP}s.
\newblock In {\em Advances in Neural Information Processing Systems},
  volume~14. The {MIT} Press, Cambridge, 2002.

\bibitem[Hertz et~al.(1991)Hertz, Krogh, and Palmer]{HerEtAl91}
J.~Hertz, A.~Krogh, and R.~G. Palmer.
\newblock {\em Introduction to the Theory of Neural Computation}.
\newblock Addison-Wesley, Reading, MA, 1991.

\bibitem[Hinton(1999)]{Hin99}
G.~E. Hinton.
\newblock Products of experts.
\newblock In {\em ICANN-99}, volume~1, pages 1--6, 1999.

\bibitem[Hinton(2002)]{Hin02}
G.~E. Hinton.
\newblock Training products of experts by minimizing contrastive divergence.
\newblock {\em Neural Computation}, 14\penalty0 (8):\penalty0 1771--1800, 2002.

\bibitem[Hinton and Sejnowski(1986)]{HinSej86}
G.~E. Hinton and T.~J. Sejnowski.
\newblock {\em Parallel Distributed Processing}, volume~1, chapter Learning and
  Relearning in Boltzman Machines, pages 282--317.
\newblock The {MIT} Press, Cambridge, 1986.

\bibitem[Howard(1960)]{How60}
R.~A. Howard.
\newblock {\em Dynamic Programming and {M}arkov Processes}.
\newblock The MIT Press, Cambridge, MA, 1960.

\bibitem[Jaakkola(1997)]{Jaa97}
T.~S. Jaakkola.
\newblock {\em Variational Methods for {I}nference and Estimation in Graphical
  Models}.
\newblock Department of Brain and Cognitive Sciences, MIT, Cambridge, MA, 1997.
\newblock Ph.D. thesis.

\bibitem[Jaakkola et~al.(1995)Jaakkola, Singh, and Jordan]{JaaEtAl95}
T.~S. Jaakkola, S.~P. Singh, and M.~I. Jordan.
\newblock Reinforcement learning algorithm for partially observable {M}arkov
  decision problems.
\newblock In Gerald Tesauro, David~S. Touretzky, and Todd~K. Leen, editors,
  {\em Advances in Neural Information Processing Systems}, volume~7, pages
  345--352. The {MIT} Press, Cambridge, 1995.

\bibitem[Jordan et~al.(1999)Jordan, Ghahramani, Jaakkola, and Saul]{JorEtAl99}
M.~I. Jordan, Z.~Ghahramani, T.~S. Jaakkola, and L.~K. Saul.
\newblock An introduction to variational methods for graphical models.
\newblock {\em Machine Learning}, 37:\penalty0 183:233, 1999.

\bibitem[Kalman(1960)]{Kal60}
R.~E. Kalman.
\newblock A new approach to linear filtering and prediction problems.
\newblock {\em Trans. ASME, Series D, Journal of Basis Engineering},
  82:\penalty0 35--45, March 1960.

\bibitem[Lovejoy(1991)]{Lov91}
W.~S. Lovejoy.
\newblock A survey of algorithmic methods for partially observable {M}arkov
  decision processes.
\newblock {\em Annals of Operations Research}, 28:\penalty0 47--66, 1991.

\bibitem[Mc{A}llester and Singh(1999)]{McASin99}
D.~A. Mc{A}llester and S.~P. Singh.
\newblock Approximate planning for factored {POMDP}s using belief state
  simplification.
\newblock In {\em Proc. UAI'99}, 1999.

\bibitem[McGovern(1998)]{McG98}
A.~McGovern.
\newblock {acQuire}-macros: An algorithm for automatically learning
  macro-actions.
\newblock In {\em NIPS98 Workshop on Abstraction and Hierarchy in Reinforcement
  Learning}, 1998.

\bibitem[Metropolis et~al.(1953)Metropolis, Rosenbluth, Rosenbluth, Teller, and
  Teller]{MetEtAl53}
N.~Metropolis, A.~W. Rosenbluth, M.~N. Rosenbluth, A.~H. Teller, and E.~Teller.
\newblock Equation of state calculations by fast computing machines.
\newblock {\em Journal of Chemical Physics}, 21:\penalty0 1087--1092, 1953.

\bibitem[Meuleau et~al.(1998)Meuleau, Hauskrecht, Kim, Peshkin, Kaelbling,
  Dean, and Boutilier]{MeuEtAl98}
N.~Meuleau, M.~Hauskrecht, K-E Kim, L.~Peshkin, L.~P. Kaelbling, T.~Dean, and
  C.~Boutilier.
\newblock Solving very large weakly coupled {M}arkov decision processes.
\newblock In {\em Proc. AAAI-98}, 1998.

\bibitem[Neal(1992)]{Nea92}
R.~M. Neal.
\newblock Connectionist learning of belief networks.
\newblock {\em Artificial Intelligence}, 56:\penalty0 71--113, 1992.

\bibitem[Neal(1993)]{Nea93}
R.~M. Neal.
\newblock Probabilistic inference using {M}arkov chain {M}onte {C}arlo methods.
\newblock Technical Report CRG-TR-93-1, Department of Computer Science,
  University of Toronto, 1993.

\bibitem[Neal and Hinton(1998)]{NeaHin98}
R.~M. Neal and G.~E. Hinton.
\newblock A view of the {EM} algorithm that justifies incremental, sparse, and
  other variants.
\newblock In M.~I. Jordan, editor, {\em Learning in Graphical Models}, pages
  355--368. Kluwer Academic Publishers, 1998.

\bibitem[Parr and Russell(1998)]{ParRus97}
R.~Parr and S.~Russell.
\newblock Reinforcement learning with hierarchies of machines.
\newblock In M.~I. Jordan, M.~J. Kearns, and S.~A. Solla, editors, {\em
  Advances in Neural Information Processing Systems}, volume~10. The {MIT}
  Press, Cambridge, 1998.

\bibitem[Peshkin et~al.(2000)Peshkin, Kim, Meuleau, and Kaelbling]{PesEtAl00}
L.~Peshkin, K-E Kim, N.~Meuleau, and L.~P. Kaelbling.
\newblock Learning to cooperate via policy search.
\newblock In {\em Proceedings of the 16th Conference on Uncertainty in
  Artificial Intelligence}, 2000.

\bibitem[Peshkin et~al.(1999)Peshkin, Meuleau, and Kaelbling]{PesEtAl99}
L.~Peshkin, N.~Meuleau, and L.~P. Kaelbling.
\newblock Learning policies with external memory.
\newblock In {\em Proc. 16th International Conference on Machine Learning},
  pages 307--314, 1999.

\bibitem[Potts(1952)]{Pot52}
R.~B. Potts.
\newblock Some generalized order-disorder transformations.
\newblock {\em Proc. Camb. Phil. Soc.}, 48:\penalty0 106--109, 1952.

\bibitem[Poupart and Boutilier(2001)]{PouBou01}
P.~Poupart and C.~Boutilier.
\newblock Vector-space analysis of belief-state approximation for {POMDP}s.
\newblock In {\em Proceedings of the Seventeenth Annual Conference on
  Uncertainty in Artificial Intelligence (UAI-01)}, 2001.

\bibitem[Precup et~al.(1998)Precup, Sutton, and Singh]{PreEtAl98}
D.~Precup, R.~Sutton, and S.~Singh.
\newblock Theoretical results on reinforcement learning with temporally
  abstract options.
\newblock In {\em Proceedings of the Tenth European Conference on Machine
  Learning}, 1998.

\bibitem[Rabiner and Juang(1986)]{Rab86}
L.~R. Rabiner and B.-H. Juang.
\newblock An introduction to hidden {M}arkov models.
\newblock {\em IEEE ASSAP Magazine}, 3:\penalty0 4--16, January 1986.

\bibitem[Rodriguez et~al.(2000)Rodriguez, Parr, and Koller]{RoPaKo00}
A.~Rodriguez, R.~Parr, and D.~Koller.
\newblock Reinforcement learning using approximate belief states.
\newblock In S.~A. Solla, T.~K. Leen, and K-R M\"{u}ller, editors, {\em
  Advances in Neural Information Processing Systems}, volume~12. The {MIT}
  Press, Cambridge, 2000.

\bibitem[Rumelhart et~al.(1986)Rumelhart, Hinton, and Williams]{RumHinWil86}
D.~E. Rumelhart, G.~E. Hinton, and R.~J. Williams.
\newblock Learning internal representations by back-propagating errors.
\newblock {\em Nature}, 323:\penalty0 533--536, 1986.

\bibitem[Rummery and Niranjan(1994)]{RumNir94}
G.~A. Rummery and M.~Niranjan.
\newblock On-line {Q}-learning using connectionist systems.
\newblock {Technical Report CUED/F-INFENG/TR 166}, Engineering Department,
  Cambridge University, 1994.

\bibitem[Sabes and Jordan(1996)]{SabJor96}
P.~N. Sabes and M.~I. Jordan.
\newblock Reinforcement learning by probability matching.
\newblock In D.~S. Touretzky, M.~C. Mozer, and M.~E. Hasselmo, editors, {\em
  Advances in Neural Information Processing Systems}, volume~8, pages
  1080--1086. The {MIT} Press, Cambridge, 1996.

\bibitem[Sallans(2000)]{Sal00}
B.~Sallans.
\newblock Learning factored representations for partially observable {M}arkov
  decision processes.
\newblock In S.~A. Solla, T.~K. Leen, and K-R M\"{u}ller, editors, {\em
  Advances in Neural Information Processing Systems}, volume~12, pages
  1050--1056. The {MIT} Press, Cambridge, 2000.

\bibitem[Santamaria et~al.(1998)Santamaria, Sutton, and Ram]{SanEtAl98}
J.~C. Santamaria, R.~S. Sutton, and A.~Ram.
\newblock Experiments with reinforcement learning in problems with continuous
  state and action spaces.
\newblock {\em Adaptive Behavior}, 6\penalty0 (2), 1998.

\bibitem[Singh et~al.(2000)Singh, Jaakkola, Littmann, and
  Szepasvfiri]{SinEtAl00}
S.~Singh, T.~Jaakkola, M.~Littmann, and C.~Szepasvfiri.
\newblock Convergence results for single-step on-policy reinforcementlearning
  algorithms.
\newblock {\em Machine Learning}, 38\penalty0 (3):\penalty0 287--308, 2000.

\bibitem[Smolensky(1986)]{Smo86}
P.~Smolensky.
\newblock Information processing in dynamical systems: Foundations of harmony
  theory.
\newblock In D.~E. Rumelhart and J.~L. McClelland, editors, {\em Parallel
  Distributed Processing: Explorations in the Microstructure of Cognition.
  Volume 1: Foundations}. The MIT Press, Cambridge, MA, 1986.

\bibitem[Sutton(1984)]{Sut84}
R.~S. Sutton.
\newblock {\em Temporal Credit Assignment in Reinforcement Learning}.
\newblock University of Massachusetts, Amherst, 1984.
\newblock Ph.D. thesis.

\bibitem[Sutton(1988)]{Sut88}
R.~S. Sutton.
\newblock Learning to predict by the method of temporal differences.
\newblock {\em Machine Learning}, 3:\penalty0 9--44, 1988.

\bibitem[Sutton(1996)]{Sut96}
R.~S. Sutton.
\newblock Generalization in reinforcement learning: Successful examples using
  sparse coarse coding.
\newblock In David~S. Touretzky, Michael~C. Mozer, and Michael~E. Hasselmo,
  editors, {\em Advances in Neural Information Processing Systems}, volume~8,
  pages 1038--1044. The {MIT} Press, Cambridge, 1996.

\bibitem[Sutton and Barto(1998)]{SutBar98}
R.~S Sutton and A.~G. Barto.
\newblock {\em Reinforcement Learning: An Introduction}.
\newblock The MIT Press, Cambridge, MA, 1998.

\bibitem[Thrun(2000)]{Thr00}
S.~Thrun.
\newblock {M}onte {C}arlo {POMDP}s.
\newblock In S.~A. Solla, T.~K. Leen, and K-R M\"{u}ller, editors, {\em
  Advances in Neural Information Processing Systems}, volume~12. The {MIT}
  Press, Cambridge, 2000.

\end{thebibliography}
