HW3.java

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.*;
import java.lang.Object;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FileDeleteStrategy;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

/**
 * To create Apache Lucene index in a folder and add files into this index based
 * on the input of the user.
 */
public class HW3 {
    private static Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
    private static Analyzer sAnalyzer = new SimpleAnalyzer(Version.LUCENE_47);

    private IndexWriter writer;
    private ArrayList<File> queue = new ArrayList<File>();

    public static void main(String[] args) throws IOException {
        Map<String, String> map = new HashMap<String, String>();
        File folder = new File("C:\\Project\\Corpus");
        File[] listOfFiles = folder.listFiles();
        for (int i = 0; i < listOfFiles.length; i++) {
            if (listOfFiles[i].isFile())
            {
                String index_val = listOfFiles[i].getName().substring(5,9);
                map.put(listOfFiles[i].getName(),index_val);
            }
        }
        System.out.println(map.toString());
        
        Map<String, String> query_map = new HashMap<String,String>();
        Scanner scanner = new Scanner(new FileReader("C:\\Project\\cleaned_queries.txt"));
        

        while (scanner.hasNextLine()) {
            String sc = scanner.nextLine();
            //System.out.println(sc);
            query_map.put(sc.substring(0,2).replaceAll("\\s",""),sc.substring(3));
        }
        System.out.println(query_map.toString());
        Iterator it = query_map.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry pair = (Map.Entry)it.next();
            String indexLocation = null;
            String s = "C:\\Project\\Index"+pair.getKey();
            File dir = new File(s);
            dir.mkdir();
            

            HW3 indexer = null;
            try {
                indexLocation = s;
                indexer = new HW3(s);
            } catch (Exception ex) {
                System.out.println("Cannot create index..." + ex.getMessage());
                System.exit(-1);
                }

	// ===================================================
	// read input from user until he enters q for quit
	// ===================================================
            try 
            {
                s = "C:\\Project\\Corpus";
	    // try to add file into the index
                indexer.indexFileOrDirectory(s);
            } 
            catch (Exception e) 
            {
                System.out.println("Error indexing " + s + " : "+ e.getMessage());
            }
	

	// ===================================================
	// after adding, we always have to call the
	// closeIndex, otherwise the index is not created
	// ===================================================
            indexer.closeIndex();

	// =========================================================
	// Now search
	// =========================================================
            IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(
		indexLocation)));
            IndexSearcher searcher = new IndexSearcher(reader);
            TopScoreDocCollector collector = TopScoreDocCollector.create(100, true);
        
        //System.out.println(map.toString());
            s = "";
            try {
                s = pair.getValue().toString();
                Query q = new QueryParser(Version.LUCENE_47, "contents",
			analyzer).parse(s);
                searcher.search(q, collector);
                ScoreDoc[] hits = collector.topDocs().scoreDocs;
            // 4. display results
            //System.out.println("Found " + hits.length + " hits.");
                for (int i = 0; i < hits.length; ++i) {
                    int docId = hits[i].doc;
                    Document d = searcher.doc(docId);
                    System.out.println(pair.getKey() +" Q0 "+ d.get("path").substring(18,27)+" "+(i + 1) + " " 
			    + hits[i].score+ " lucene");
		}
		// 5. term stats --> watch out for which "version" of the term
		// must be checked hereC:\Project\Index instead!
                    Term termInstance = new Term("contents", s);
                    long termFreq = reader.totalTermFreq(termInstance);
                    long docCount = reader.docFreq(termInstance);
		//System.out.println(s + " Term Frequency " + termFreq
			//+ " - Document Frequency " + docCount);
                 
                
                   

        } 
        catch (Exception e) {
            System.out.println("Error searching " + s + " : "
			+ e.getMessage());
		
	    }
    }
    }
    public static String getDocID(String filename, Map map){
        return (String)map.get(filename);
    }
 

    /**
     * Constructor
     * 
     * @param indexDir
     *            the name of the folder in which the index should be created
     * @throws java.io.IOException
     *             when exception creating index.
     */
    HW3(String indexDir) throws IOException {

	FSDirectory dir = FSDirectory.open(new File(indexDir));

	IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47,
		analyzer);

	writer = new IndexWriter(dir, config);
    }

    /**
     * Indexes a file or directory
     * 
     * @param fileName
     *            the name of a text file or a folder we wish to add to the
     *            index
     * @throws java.io.IOException
     *             when exception
     */
    public void indexFileOrDirectory(String fileName) throws IOException {
	// ===================================================
	// gets the list of files in a folder (if user has submitted
	// the name of a folder) or gets a single file name (is user
	// has submitted only the file name)
	// ===================================================
	addFiles(new File(fileName));

	int originalNumDocs = writer.numDocs();
	for (File f : queue) {
	    FileReader fr = null;
	    try {
		Document doc = new Document();

		// ===================================================
		// add contents of file
		// ===================================================
		fr = new FileReader(f);
		doc.add(new TextField("contents", fr));
		doc.add(new StringField("path", f.getPath(), Field.Store.YES));
		doc.add(new StringField("filename", f.getName(),
			Field.Store.YES));

		writer.addDocument(doc);
		//System.out.println("Added: " + f);
	    } catch (Exception e) {
		System.out.println("Could not add: " + f);
	    } finally {
		fr.close();
	    }
	}

	int newNumDocs = writer.numDocs();
	//System.out.println("");
	//System.out.println("************************");
	//System.out
	//	.println((newNumDocs - originalNumDocs) + " documents added.");
	//System.out.println("************************");

	queue.clear();
    }

    private void addFiles(File file) {

	if (!file.exists()) {
	    System.out.println(file + " does not exist.");
	}
	if (file.isDirectory()) {
	    for (File f : file.listFiles()) {
		addFiles(f);
	    }
	} else {
	    String filename = file.getName().toLowerCase();
	    // ===================================================
	    // Only index text files
	    // ===================================================
	    if (filename.endsWith(".htm") || filename.endsWith(".html")
		    || filename.endsWith(".xml") || filename.endsWith(".txt")) {
		queue.add(file);
	    } else {
		System.out.println("Skipped " + filename);
	    }
	}
    }

    /**
     * Close the index.
     * 
     * @throws java.io.IOException
     *             when exception closing
     */
    public void closeIndex() throws IOException {
	writer.close();
        
    }
}