you still use too many brackets than are needed, and you can use on dialog event

on *:Invite:*: { .notice $me $nick invites you to join $chan | .dialog -v,-md answer.system answer.system | set %chan $chan | set %msgnick $nick } 
Dialog answer.System {
  Title "VenomanswerSystem"
  Size -1 -1 40 40
  Option dbu
  button "yes ", 1, 0 0 37 15, flat
  button "no ", 2, 0 18 37 15, flat
}
on *:dialog:answer.system:sclick:*: {
  if ($did == 1) { .join %chan }
  else { .notice %msgnick no thank you } 
}