<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40"><head><META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=us-ascii"><meta name=Generator content="Microsoft Word 15 (filtered medium)"><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Verdana;
panose-1:2 11 6 4 3 5 4 4 2 4;}
@font-face
{font-family:"Open Sans";
panose-1:0 0 0 0 0 0 0 0 0 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
span.EmailStyle18
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle21
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.apple-converted-space
{mso-style-name:apple-converted-space;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1565338933;
mso-list-template-ids:-363824088;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:"Courier New";
mso-bidi-font-family:"Times New Roman";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1
{mso-list-id:1671789125;
mso-list-template-ids:240390404;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]--></head><body lang=EN-US link="#0563C1" vlink="#954F72"><div class=WordSection1><p class=MsoNormal>Hi everyone,<o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal>Please note that our first deadline is today but we have a second deadline on July 17.<o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal style='background:white'><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:black'>Papers submitted to the workshop should be up to four pages long excluding references and in<span class=apple-converted-space> </span><a href="https://2017.icml.cc/Conferences/2017/StyleAuthorInstructions"><span style='color:#835EA5'>ICML 2017 format</span></a>. <o:p></o:p></span></p><p class=MsoNormal style='background:white'><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:black'>*<b>Authors with a relevant submission to the main conference are invited to submit their paper as is.</b>* </span><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:#616161'><o:p></o:p></span></p><p class=MsoNormal style='background:white'><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:#616161'><o:p> </o:p></span></p><p class=MsoNormal style='background:white'><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:black'>Papers can be submitted through EasyChair: </span><b><span style='font-size:10.0pt;font-family:"Verdana",sans-serif;color:black'>https://easychair.org/conferences/?conf=rmlw17. </span></b><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:black'>As the review process is not blind, authors can reveal their identity in their submissions. Accepted submissions will be presented as posters or talks. </span><span style='font-size:10.5pt;font-family:"Open Sans",serif;color:#616161'><o:p></o:p></span></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal>Thanks,<o:p></o:p></p><p class=MsoNormal>Adrian<o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><div><div style='border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal><b>From:</b> Adrian Weller [mailto:aw665@cam.ac.uk] <br><b>Sent:</b> Thursday, June 8, 2017 5:13 PM<br><b>Cc:</b> 'Dylan Hadfield-Menell' <dhm@eecs.berkeley.edu>; 'Jacob Steinhardt' <jacob.steinhardt@gmail.com>; 'Smitha Milli' <smilli@berkeley.edu><br><b>Subject:</b> CfP: Reliable Machine Learning in the Wild at ICML 2017, deadline 16 June or 17 July<o:p></o:p></p></div></div><p class=MsoNormal><o:p> </o:p></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><span style='color:black'>Call for Papers for ICML 2017 Workshop on Reliable Machine Learning in the Wild, please forward to others who may have interest.</span><o:p></o:p></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><o:p> </o:p></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><span style='color:black'>Workshop website </span><a href="https://sites.google.com/site/wildml2017icml/"><span style='color:#1155CC'>https://sites.google.com/site/wildml2017icml/</span></a><span style='color:black'> </span><o:p></o:p></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'> <o:p></o:p></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><span style='color:black'>When can we trust that a system that has performed well in the past will continue to do so in the future? Designing systems that are reliable in the wild is essential for high stakes applications such as self-driving cars and automated surgical assistants. This workshop aims to bring together researchers in diverse areas such as reinforcement learning, human-robot interaction, game theory, cognitive science, and security to further the field of reliability in machine learning. We will focus on three aspects — robustness (to adversaries, distributional shift, model misspecification, corrupted data); awareness (of when a change has occurred, when the model might be miscalibrated, etc.);and adaptation (to new situations or objectives). We aim to consider each of these in the context of the complex human factors that impact the successful application or meaningful monitoring of any artificial intelligence technology. Together, these will aid us in designing and deploying reliable machine learning systems.</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'> <o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>We are seeking submissions that deal with the challenges of reliably applied machine learning techniques in the real world. Some possible questions touching on each of these categories are given below, though we also welcome submissions that do not directly fit into these categories.</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'> <o:p></o:p></p><ul style='margin-top:0in' type=disc><li class=MsoNormal style='color:black;margin-top:14.0pt;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Robustness: </b>How can we make a system robust to novel or potentially adversarial inputs? What are ways of handling model mis-specification or corrupted training data? What can be done if the training data is potentially a function of system behavior or of other agents in the environment (e.g. when collecting data on users that respond to changes in the system and might also behave strategically)?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Awareness: </b>How do we make a system aware of its environment and of its own limitations, so that it can recognize and signal when it is no longer able to make reliable predictions or decisions? Can it successfully identify “strange” inputs or situations and take appropriately conservative actions? How can it detect when changes in the environment have occurred that require re-training? How can it detect that its model might be mis-specified or poorly-calibrated?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Adaptation:</b> How can machine learning systems detect and adapt to changes in their environment, especially large changes (e.g. low overlap between train and test distributions, poor initial model assumptions, or shifts in the underlying prediction function)? How should an autonomous agent act when confronting radically new contexts?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Monitoring: </b>How can we monitor large-scale systems in order to judge if they are performing well? If things go wrong, what tools can help?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Value Alignment:</b> For systems with complex desiderata, how can we learn a value function that captures and balances all relevant considerations? How should a system act given uncertainty about its value function? Can we make sure that a system reflects the values of the humans who use it?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Reward Hacking: </b>How can we ensure that the objective of a system is immune to <i>reward hacking? </i>Reward hacking is a way that the system can attain high reward that was unintended by the system designer. For example see <a href="https://blog.openai.com/faulty-reward-functions/">https://blog.openai.com/faulty-reward-functions/</a><b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li><li class=MsoNormal style='color:black;margin-left:0in;mso-list:l0 level1 lfo3;vertical-align:baseline'><b>Human Factors:</b> Actual humans will be interacting and adapting to these systems when they are deployed. How do properties of humans affect the guarantees of performance that the system has? What if the humans are suboptimal or even adversarial?<b><span style='font-family:"Arial",sans-serif'><o:p></o:p></span></b></li></ul><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><span style='color:black'><o:p> </o:p></span></p><p style='mso-margin-top-alt:14.0pt;margin-right:0in;margin-bottom:0in;margin-left:0in;margin-bottom:.0001pt'><span style='color:black'>How to submit</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Papers submitted to the workshop should be up to four pages long excluding references and in</span><a href="https://2017.icml.cc/Conferences/2017/StyleAuthorInstructions"><span style='color:black'> </span><span style='color:#1155CC'>ICML 2017 format</span></a><span style='color:black'>. They should be submitted via Easychair at </span><a href="https://easychair.org/conferences/?conf=rmlw17"><span style='color:#1155CC'>https://easychair.org/conferences/?conf=rmlw17</span></a><span style='color:black'> . As the review process is not blind, authors can reveal their identity in their submissions. Accepted submissions will be presented as posters or talks.</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'> <o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>We will accept submissions at two deadlines. One earlier deadline, with an earlier acceptance notification, and one later one. Our goal is to allow for late submission to the extent that we can, while still allowing some people to get early confirmation of paper acceptance, which they might need in order to arrange travel in time.</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'> <o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><b><span style='color:black'>Important Dates:</span></b><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Submission deadline 1: 16 June 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Acceptance notification 1: 1 July 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Submission deadline 2: 17 July 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Acceptance notification 2: 31 July 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Final camera-ready versions of accepted papers: 5 August 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'><span style='color:black'>Workshop: 11 August 2017</span><o:p></o:p></p><p style='margin:0in;margin-bottom:.0001pt'> <o:p></o:p></p><p class=MsoNormal><span style='color:black'>Thank you,</span><o:p></o:p></p><p class=MsoNormal><span style='color:black'>Dylan, Jacob, Smitha and Adrian</span><o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal>----------------------------------------------<o:p></o:p></p><p class=MsoNormal>Adrian Weller<o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p></div></body></html>