# remove duplicates in file
awk '!a[$0]++' input.txt