Skip to content

Commit

Permalink
removing database normalizing feature
Browse files Browse the repository at this point in the history
  • Loading branch information
nikit-srivastava committed Oct 18, 2018
1 parent 4e2b02e commit 112ef8e
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 164 deletions.
73 changes: 0 additions & 73 deletions src/main/java/org/aksw/word2vecrestful/tool/ModelNormalizer.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
import java.sql.PreparedStatement;
import java.sql.SQLException;

import org.aksw.word2vecrestful.utils.Cfg;
Expand Down Expand Up @@ -97,58 +96,6 @@ public void generateNormalizedModel(File inputFile, File outputFile) throws IOEx
}
}

/**
* Method to persist a normalized model for a word2vec bin model
*
* @param inputFile
* - word2vec file of the model to be normalized
* @param dbName
* - name of the database
* @param tablName
* - name of the table to store the data in
* @throws IOException
* @throws SQLException
*/
public void persistNormalizedModel(File inputFile, String dbName, String tblName) throws IOException, SQLException {
// intialize handler instance
NormalizedDBModelGenerator dbHandler = null;

FileInputStream fin = null;
try {
// reads file header
fin = new FileInputStream(inputFile);
String word = Word2VecModelLoader.readWord(fin);
int words = Integer.parseInt(word);
word = Word2VecModelLoader.readWord(fin);
int vectorSize = Integer.parseInt(word);
dbHandler = new NormalizedDBModelGenerator(dbName, tblName, vectorSize);
// open connection
dbHandler.connect();
LOG.info("Expecting " + words + " words with " + vectorSize + " values per vector.");
// create preparedstatement
PreparedStatement ps = dbHandler.generateMainTblInsrtStmnt();
for (int w = 0; w < words; ++w) {
word = Word2VecModelLoader.readWord(fin);
// LOG.info(word);
float[] vector = Word2VecModelLoader.readVector(fin, vectorSize);
// dbHandler.insertMainTblRecord(word, vector);
dbHandler.addMainTblInsrtBatch(word, Word2VecMath.normalize(vector), ps);
if ((w + 1) % 50000 == 0) {
dbHandler.executeBatchCommit(ps);
LOG.info((w + 1) + " Records inserted.");
}
}
dbHandler.executeBatchCommit(ps);
// Generate Index on completion
dbHandler.makeIndex();
} catch (final IOException e) {
LOG.error(e.getLocalizedMessage(), e);
} finally {
fin.close();
dbHandler.disconnect();
}
}

/**
* Method to generate a normalized model for a word2vec bin model
*
Expand Down Expand Up @@ -199,26 +146,6 @@ public void generateNormalizedBinModel(File inputFile, File outputFile) throws I
}
}

/*
* public static void main(String[] args) throws IOException { String
* cfgKeyModel = Word2VecFactory.class.getName().concat(".model"); String model
* = (Cfg.get(cfgKeyModel)); ModelNormalizer modelNormalizer = new
* ModelNormalizer(); File inputFile = new File(model); File outputFile = new
* File(
* "D:\\Nikit\\DICE-Group\\Jword2vec\\data\\normal\\GoogleNews-vectors-negative300-normalized.txt"
* ); modelNormalizer.generateNormalizedModel(inputFile, outputFile); }
*/

/*
* public static void main(String[] args) throws IOException, SQLException {
* String cfgKeyModel = Word2VecFactory.class.getName().concat(".model"); String
* model = (Cfg.get(cfgKeyModel)); ModelNormalizer modelNormalizer = new
* ModelNormalizer(); File inputFile = new File(model); //
* modelNormalizer.generateNormalizedModel(inputFile, outputFile);
* modelNormalizer.persistNormalizedModel(inputFile,
* "data/nrmldb/word2vecmodel", "wordtovec"); }
*/

public static void main(String[] args) throws IOException, SQLException {
String cfgKeyModel = Word2VecFactory.class.getName().concat(".model");
String model = (Cfg.get(cfgKeyModel));
Expand Down

This file was deleted.

0 comments on commit 112ef8e

Please sign in to comment.