About sequential learning (or interference)

[Lorien Y. Pratt]

Deliang,

My own work on transfer between neural networks has addressed the sequential 
learning problem, see pratt-93, cited below, and lots of other papers available 
from my web page: http://vita.mines.colorado.edu:3857/1s/lpratt.  My formuation 
differs from others in that, if the new task is essentially different from the 
old task, I intentionally lose old-task performance if necessary to improve on 
the new task.  Other people who have looked at a similar formulation include 
sharkey-92, sharkey-93, naik-92b, and agarwal-92, martin-88.  Those who have 
attempted to preserve old-task performance include mccloskey-89, and recent 
work by Thrun & Mitchell, especially thrun-93a, but also including thrun-92, 
thrun-93b. Sebastian and I had a long talk at NIPS about a general formulation 
for transfer as a solution to the sequential learning problem -- he's got an 
excellent formulation along these lines.

Approaches to modular training can also be viewed as one approach to handling 
the sequential learning problem, in the special case where the old tasks can be 
viewed as subtasks of the new task (i.e. pratt-91, waibel-89, and Jacobs' and 
others' recent work on mixtures of experts, including Pomerleau's approach to 
Alvinn, see last year's NIPS).

My thesis (see my web page) has a more detailed review of all this stuff.

Hope that helps.
  --Lori, recovering from NIPS
  

  @incollection{ pratt-93,
MYKEY           = " pratt-93 : .con .bap",
EDITOR          = "C.L. Giles and S. J. Hanson and J. D. Cowan",
BOOKTITLE       = "{Advances in Neural Information Processing Systems 5}",
AUTHOR          = "L. Y. Pratt",
TITLE           = "Discriminability-Based Transfer between Neural Networks",
ADDRESS         = "San Mateo, CA",
PUBLISHER       = "Morgan Kaufmann Publishers",
YEAR            = 1993,
PAGES           = {204--211},
CATALOGDATE     = "April 12, 1993",
NOTE            = "Also available via anonymous ftp to
franklinite.mines.colorado.edu: pub/pratt-papers/pratt-nips5.ps.Z",
}
@inproceedings{ sharkey-92,
MYKEY           = " sharkey-92 : ",
TITLE           = "Adaptive generalisation and the transfer of knowledge",
AUTHOR          = "Noel E. Sharkey and Amanda J. C. Sharkey",
BOOKTITLE       = "Proceedings of the Second Irish Neural Networks Conference,
                   Belfast",
YEAR            = 1992,
CATALOGDATE     = "December 18, 1992",
}

@article{ sharkey-93,
MYKEY           = " sharkey-93 : ",
TITLE           = "Adaptive generalisation and the transfer of knowledge",
AUTHOR          = "N. E. Sharkey and A. J. C. Sharkey",
NUMBER          = "In press",
VOLUME          = "Special Issue on Connectionism",
JOURNAL         = "AI Review",
YEAR            = 1993,
ANNOTE          = "In press",
CATALOGDATE     = "May 19, 1993",
}
@inproceedings{ naik-92b,
MYKEY           = " naik-92b : .eml .unb .con ",
AUTHOR          = "D. K. Naik and R. J. Mammone and A. Agarwal",
TITLE           = "Meta-Neural Network approach to learning by learning",
YEAR            = 1992,
BOOKTITLE       = "Intelligence Engineering Systems through Artificial Neural
                   Networks",
ORGANIZATION    = "The American Society of Mechanical Engineers",
PUBLISHER       = "ASME Press",
VOLUME          = 2,
PAGES           = {245--252},
CATALOGDATE     = "December 23, 1992",
}

@inproceedings{ agarwal-92,
MYKEY           = " agarwal-92 : .eml .unb .con ",
AUTHOR          = "A. Agarwal and R. J. Mammone and D. K. Naik",
TITLE           = "An on-line Training Algorithm to Overcome Catastrophic
                   Forgetting",
BOOKTITLE       = "Intelligence Engineering Systems through Artificial Neural
                   Networks",
YEAR            = 1992,
ORGANIZATION    = "The American Society of Mechanical Engineers",
PUBLISHER       = "ASME Press",
VOLUME          = 2,
PAGES           = {239--244},
CATALOGDATE     = "December 23, 1992",
}

@techreport{ martin-88,
MYKEY           = " martin-88 : ",
TITLE           = "The Effects of Old Learning on New in Hopfield and
                   Backpropagation Nets",
AUTHOR          = "Gale Martin",
KEY             = "martin-88",
INSTITUTION     = "Microelectronics and Computer Technology Corporation (MCC)",
NUMBER          = "ACA-HI-019",
YEAR            = 1988,
CATALOGDATE     = "January 3, 1992",
}

@article{ mccloskey-89,
MYKEY           = " mccloskey-89 : .unb .con .csy .adap",
TITLE           = "Catastrophic interference in connectionist networks: the
                   sequential learning problem",
KEY             = "mccloskey-89",
AUTHOR          = "Michael McCloskey and Neal J. Cohen",
JOURNAL         = "The psychology of learning and motivation",
VOLUME          = 24,
YEAR            = 1989,
CATALOGDATE     = "April 9, 1991",
}
@inproceedings{ thrun-93a,
MYKEY           = " thrun-93a : ",
TITLE           = "Lifelong Robot Learning",
AUTHOR          = "Sebastian B. Thrun and Tom M. Mitchell",
BOOKTITLE       = "Proceedings of the {NATO} {ASI}: The biology and technology
                   of intelligent autonomous agents",
EDITOR          = "Luc Steels",
YEAR            = 1993,
KEY             = "thrun-93a",
CATALOGDATE     = "September 1, 1993",
}

@INPROCEEDINGS{thrun-93b,
AUTHOR        = {Thrun, Sebastian B. and Mitchell, Tom M.},
TITLE         = {Integrating Inductive Neural Network Learning and
                 Explanation-Based Learning},
BOOKTITLE     = {Proceedings of IJCAI-93},
YEAR          = {1993},
ORGANIZATION  = {IJCAI, Inc.},
PUBLISHER     = {},
ADDRESS       = {Chamberry, France},
MONTH         = {},
CATALOGDATE     = "October 11, 1993",
}

@inproceedings{ pratt-91,
MYKEY           = " pratt-91 : .min .bap .app .spc .con ",
AUTHOR          = "Lorien Y. Pratt and Jack Mostow and Candace A. Kamm",
TITLE           = "Direct Transfer of Learned Information among Neural
                    Networks",
BOOKTITLE       = "Proceedings of the Ninth National Conference on
                   Artificial Intelligence (AAAI-91)",
PAGES           = {584--589},
ADDRESS         = "Anaheim, CA",
YEAR            = 1991,
}

@article{ waibel-89,
MYKEY           = " waibel-89 : .bap .unr .unb .tem .spc .con ",
TITLE           = "Modular Construction of Time-Delay Neural Networks for
                    Speech Recognition",
AUTHOR          = "Alexander Waibel",
journal         = "Neural Computation",
volume          = 1,
pages           = {39--46},
year            = 1989
}