src/filer.js


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

const fs = require('fs');
const api = require('./api');

function pad(n, z) {
    z = z || 2;
    return ('0000000000' + n).slice(-z);
}

function msToTime(s) {
    var ms = s % 1000;
    s = (s - ms) / 1000;
    var secs = s % 60;
    s = (s - secs) / 60;
    var mins = s % 60;
    var hrs = (s - mins) / 60;

    return pad(hrs) + ':' + pad(mins) + ':' + pad(secs)
}

module.exports = (lang) => {
    console.log("Reading files at /train/" + lang + "/...");
    files = fs.readdirSync("./train/" + lang);

    files.forEach((file, index) => {
        console.log("Processing file /train/" + lang + "/" + file);
        content = fs.readFileSync("./train/" + lang + "/" + file).toString();
        sentences = content.match(/([^\.!\?]+[\.!\?]+)|([^\.!\?]+$)/g);
        total = sentences.length;
        done = 0;
        sentences.forEach((e) => {
            ccold = new Date();
            api(e, lang, true);
            ccrec = new Date();
            ccdiff = ccrec - ccold;
            if ((done / 50) == Math.round(done / 50)) {
                cceta = ccdiff * (total - done);
            }
            console.log(lang + " - " + pad(done, total.toString().length) + "/" + total + " (" + pad(Math.round((done/total)*100), 3) + "%, " + pad(total - done, total.toString().length) + ") - " + pad(index + 1, files.length.toString().length) + "/" + files.length + " - " + msToTime(cceta) + " - " + msToTime((cceta) * (__train_totallangs - __train_donelangs)));
            done++;
        })
    })
}