求助（java代码），VSM空间表示模型，IF-IDF计算有问题吗？请帮忙看一下
0 1:60 3:72 4:18 5:1 6:3 7:6 8:1 9:1 10:1 12:1 13:1 14:4 15:11 16:1 18:8 20:4 23:11 25:9 26:3 27:1 32:1 33:4 41:2 45:2 46:5 47:1 49:2 51:26 57:2 59:1 67:1 70:2 74:1 75:4 77:3 78:10 79:37 80:1 82:1 83:3 86:3 96:2 99:1上面是经过特征选择和统计计算得到的特征项和对应的词频数，但是经过IF-IDF后，计算结果有些词的权重却为0，不知道那有问题，帮忙看一下吧！
0 1:0.521 3:0.697 4:0.189 5:0.000 6:0.000 7:0.063 8:0.004 9:0.004 10:0.000 12:0.004 13:0.000 14:0.045 15:0.123 16:0.004 18:0.030 20:0.000 23:0.041 25:0.112 26:0.018 27:0.000 32:0.006 33:0.052 41:0.012 45:0.007 46:0.000 47:0.004 49:0.007 51:0.097 57:0.007 59:0.013 67:0.009 70:0.028 74:0.000 75:0.042 77:0.026 78:0.000 79:0.389 80:0.007 82:0.006 83:0.040 86:0.037 96:0.012 99:0.012
代码如下：
public class VsmModel {
//文档总数
private int totalDoc = 0;
//每一个Term对应的文章数；
private Map<String, Integer> termMap = new HashMap<String, Integer>();
public void getSource(File dataFile , File newDataFile){
String dataFileName = dataFile.getName();
String labelFileName = dataFileName.substring(0, dataFileName.indexOf("."))
+"."+Parameter.labelSuff;
String termFileName = dataFileName.substring(0, dataFileName.indexOf("."))
+"."+Parameter.termSuff;
String mapFileName = dataFileName.substring(0, dataFileName.indexOf("."))
+"."+Parameter.mapSuff;
File labelFile = new File(dataFile.getParent(),labelFileName);
File termFile = new File(dataFile.getParent(),termFileName);
File mapFile = new File(dataFile.getParent(),mapFileName);
String newdataFileName = newDataFile.getName();
String newlabelFileName = newdataFileName.substring(0, newdataFileName.indexOf("."))
+"."+Parameter.labelSuff;
String newtermFileName = newdataFileName.substring(0, newdataFileName.indexOf("."))
+"."+Parameter.termSuff;
String newmapFileName = newdataFileName.substring(0, newdataFileName.indexOf("."))
+"."+Parameter.mapSuff;
File newlabelFile = new File(newDataFile.getParent(),newlabelFileName);
File newtermFile = new File(newDataFile.getParent(),newtermFileName);
File newmapFile = new File(newDataFile.getParent(),newmapFileName);
if(Parameter.corpusMethod == Parameter.TfIdf){
this.getLabel(labelFile);
this.getTerm(termFile);
this.calTfIdf(dataFile, newDataFile);
}
if(Parameter.corpusMethod == Parameter.TF){
this.copyFile(dataFile, newDataFile);
}
this.copyFile(labelFile, newlabelFile);
this.copyFile(termFile, newtermFile);
this.copyFile(mapFile, newmapFile);
}
private void getLabel(File labelFile){
try {
FileReader fr = new FileReader(labelFile);
BufferedReader br=new BufferedReader(fr);
String line = br.readLine();
int start = 0;
StringBuffer labelInfo = new StringBuffer();
while((line = br.readLine())!=null){
labelInfo.append(line.split(" ",2)[1]);
start = labelInfo.indexOf(":");
this.totalDoc += Integer.parseInt(labelInfo.substring(0, start));
labelInfo = new StringBuffer();
}
System.out.println("类别信息载入完成！");
br.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void getTerm(File termFile){
try {
FileReader fr = new FileReader(termFile);
BufferedReader br=new BufferedReader(fr);
String line = "";
String term = "";
String[] lineSplit = null;
int length = 0;
int start = 0,end =0;
StringBuffer indexInfo = new StringBuffer();
while((line = br.readLine())!=null){
int termTotalDoc = 0;
lineSplit = line.split(" ");
length = lineSplit.length;
term = lineSplit[0].trim();
for(int i = 1;i<length ; i++){
indexInfo.append(lineSplit[i]);
start = indexInfo.indexOf(":");
end = indexInfo.lastIndexOf(":");
if(start<1 && end<1){
System.err.println((new StringBuilder("wrong in ")).append(indexInfo.toString()).toString());
continue;
}
termTotalDoc += Integer.parseInt(indexInfo.substring(start+1, end));
indexInfo = new StringBuffer();
}
termMap.put(term, termTotalDoc);
}
br.close();
System.out.println("特征信息载入完成！");
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void copyFile(File originalFile , File newFile){
try {
FileReader fr = new FileReader(originalFile);
BufferedReader br=new BufferedReader(fr);
FileWriter fw = new FileWriter(newFile);
BufferedWriter bw = new BufferedWriter(fw);
String line = "";
while((line = br.readLine())!=null){
bw.append(line);
bw.newLine();
}
bw.close();
br.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void calTfIdf(File dataFile , File newDataFile){
System.out.println("开始计算Tf*IDF.....");
try {
FileReader fr = new FileReader(dataFile);
BufferedReader testbr=new BufferedReader(fr);
FileWriter fw = new FileWriter(newDataFile);
BufferedWriter bw = new BufferedWriter(fw);
DecimalFormat df = new DecimalFormat("0.000");
String line = "";
int lineNum = 0;
int type = 0;
StringBuffer newLine = new StringBuffer();
String[] splitLine = null;
while((line = testbr.readLine())!= null){
Map<Integer,Double> termValueMap = new HashMap<Integer,Double>();
double totalWeight = 0.0D;
splitLine = line.split(" ");
if(splitLine.length== 0 || !splitLine[0].toString().matches("\\d+")){
System.err.println(lineNum+"is wrong!");
continue;
}
int key , value, numberTerm ;
double termWeight;
for(int i = 1; i<splitLine.length ; i++){
String[] node = splitLine[i].split(":",2);
key = Integer.parseInt(node[0]);
value = Math.round(Float.parseFloat(node[1]));
if(!this.termMap.containsKey(node[0])){
System.out.println(node[0]);
}
numberTerm = ((Integer)termMap.get(node[0])).intValue();
termWeight = (double)value * Math.log((double)(totalDoc / numberTerm) + 0.0001D);
termValueMap.put(Integer.valueOf(key), Double.valueOf(termWeight));
totalWeight += termWeight*termWeight;
}
Set<Entry<Integer,Double>> termSet = termValueMap.entrySet();
for(Entry<Integer,Double> temp:termSet){
key = temp.getKey();
termWeight = temp.getValue();
termWeight = termWeight/Math.sqrt(totalWeight);
termValueMap.put(key, termWeight);
}
List<Map.Entry<Integer, Double>> mappingList = new ArrayList<Map.Entry<Integer, Double>>(termValueMap.entrySet());
Collections.sort(mappingList, new Comparator<Map.Entry<Integer,Double>>(){
public int compare(Map.Entry<Integer,Double> mapping1,Map.Entry<Integer,Double> mapping2){
return mapping1.getKey().compareTo(mapping2.getKey());
}
});
newLine.append(splitLine[0]);
for(Map.Entry<Integer, Double> temp:mappingList){
newLine.append((new StringBuilder(" ")).append(temp.getKey()).append(":").append(df.format(temp.getValue())).toString());
}
bw.append(newLine.toString());
bw.newLine();
newLine = new StringBuffer();
}
bw.close();
testbr.close();
System.out.println("tf*idf计算完成！");
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
补充：Java , Eclipse
