mkfifo fifo && sudo nc -l -u -p 123 < fifo | ssh user@server 'nc -u localhost 123' > fifo
I also had to tell ntpdate to not use 123 as the source port
ntpdate -u localhost