Bielie, you can use if ($1 isnum 1000000-999999999), also your script will fail if the calculation includes spaces. Remove chr(32). And you are only calling %output once,
$iif($left($1,1) == @,/msg #,/notice $nick) $num(%calc) $iif($short(%scalc),$v1,$chr(32)) would work. You can even leave the $chr(32), just $iif($short(%scalc),$v1)

#calc on
on $*:TEXT:/^[!.@]calc(ulate)?\w/Si:#: {
  if (!$2) { notice $nick Please use the correct syntax: !calc <calculation> | halt }
if ($regex($1-,/[^1-9][mbkt]/i)) { notice $nick Invalid expression | halt }
if ($regex($remove($2-,m,b,k,t),/\D/)) { notice $nick Invalid expression | halt }
  var %calc $calc($replace($remove($2-,$chr(32)), :, /, x, *,t,*1000b,b,*1000m,m,*1000m,k,*1000))
 $iif($left($1,1) == @,/msg #,/notice $nick) 4=2 $bytes(%calc,bd) $iif($short(%scalc),$chr(40) $+ $v1),)
  .ignore -u3 $nick
}

alias short {
  if ($1 < 0) { scon 0 return - $!+ $!short( $abs($1) ) }
  if ($1 isnum 1000-999999) { return 3( $+ $round($calc($1- / 1000),2) $+ k $+ 3) }
  if ($1 isnum 1000000-999999999) { return 3( $+ $round($calc($1- / 1000000),2) $+ m $+ 3) }
  if ($1 isnum 1000000000-999999999999) { return 3( $+ $round($calc($1- / 1000000000),2) $+ b $+ 3) }
  if ($1 >= 1000000000000) { return 3( $+ $round($calc($1- / 1000000000000),2) $+ t $+ 3) }
}
#calc end

Not waterproof but works in most cases