Ok, have a look at what i did to the events compared to yours.

alias masskill {
  if ($1) set %kill.chan $1
  else set %kill.chan $active
  dialog -mado killkline killkline
}
dialog killkline {
  title \"Mass Kill/K-Line\"
  size -1 -1 153 184
  option dbu
  box \"Don\'t Kill/K-line\", 1, 3 2 71 135
  list 2, 6 10 65 124, size vsbar
  box Kill and K-line, 3, 78 2 70 135
  list 4, 81 10 63 123, size vsbar
  edit \"Reason\", 5, 4 156 143 10
  button \"Perform\", 6, 55 169 37 12
  button \"->\", 7, 6 137 15 12
  button \"ALL\", 8, 30 137 19 12
  button \"ALL\", 9, 107 137 17 12
  button \"<-\", 10, 136 137 11 12
}
on *:JOIN:%kill.chan: if ($dialog(killkline)) { did -a killkline 2 $nick }
on *:PART:%kill.chan: {
  if ($dialog(killkline)) {
    var %a $did(killkline,2).lines
    while (%a) {
      if ($did(killkline,2,%a).text == $nick) { did -d killkline 2 %a }
      dec %a
    }
    var %b $did(killkline,4).lines
    while (%b) {
      if ($did(killkline,4,%b).text == $nick) { did -d killkline 4 %b }
      dec %b
    }
  }
}
on *:NICK: {
  if ($dialog(killkline)) {
    var %a $did(killkline,2).lines
    while (%a) {
      echo -a 1 $nick $newnick
      if ($did(killkline,2,%a).text == $nick) { did -o killkline 2 %a $newnick }
      dec %a
    }
    var %b $did(killkline,4).lines
    while (%b) {
      if ($did(killkline,4,%b).text == $nick) { did -o killkline 4 %b $newnick }
      dec %b
    }
  }
}
on *:QUIT: {
  if ($dialog(killkline)) {
    var %a $did(killkline,2).lines
    while (%a) {
      if ($did(killkline,2,%a).text == $nick) { did -d killkline 2 %a $nick }
      dec %a
    }
    var %b $did(killkline,4).lines
    while (%b) {
      if ($did(killkline4,%b).text == $nick) { did -d killkline 4 %b $nick }
      dec %b
    }
  }
}

You werent using -o to overwrite instead of deleting and adding and you also forgot to tell it which line to change in the id.

:)