#!/bin/bash

cat ./Data/*.txt > corpus.txt

tr "[a-zA-Z\:\;\.\,\'\"\`\-]" " " < corpus.txt > stage1.txt

cat stage1.txt | tr " " "\n" | sort | uniq -c | sort -gr > stage2.txt

cat stage2.txt | sed -r '/^\ *[0-9]*\ \t/d' | sed -r '/^\ *[0-9]+$/d' > stage3.txt
sed -r '/^\ *[0-9]*\ .{1,3}$/d' < stage3.txt > stage4.txt
sed -r '/^\ *[1-2]\ /d' < stage4.txt > stage5.txt
sed -r 's/^\ +//' < stage5.txt > final.txt

cd FinalDistribution
g++ UnsupervisedWordSegmentation.cpp
cd ..

./FinalDistribution/a.exe final.txt 1 1 0 1 0

cp FinalDistribution/Output/finalSegmentation.txt ./

while read line
do
	words=( $line )
	sed -rn "/^${words[1]}:/ p" < finalSegmentation.txt | head -n 1
done < list22.txt > answer.txt