1) sort -u original > de-duplicated
2) sort original | uniq > de-duplicated
3) awk '!x[$0]++' original > de-duplicated