Skip to content

Проблема при асинхронном импорте #28

Open
@akudelka

Description

@akudelka

script:

var libs = process.cwd() + '/libs/';
var fs = require('fs');
var readline = require('linebyline');
var TarantoolConnection = require('tarantool-driver');
var conn = new TarantoolConnection({port: 3301, log: false});

var inserted = 0;
conn.connect()
    .then(function() {
            var rl = readline('./libs/model/segment/dictionary.csv');

            rl.on('line', function(line, lineCount) {
                if (lineCount === 1) return;
                if (lineCount % 1000 === 0){
                    console.log('readed :', lineCount);
                }
                var tags = line.split(';');
                
                conn.insert('tags',
                        [parseInt(tags[0]), tags[1], parseInt(tags[2]), tags[5], parseInt(tags[6]), parseFloat(tags[3]), parseInt(tags[4]), 7, 5, 1]
                    ).then(function(){
                        inserted++;
                        if (inserted % 1000 === 0){
                            console.log('inserted to db :', inserted);
                        }
                    }).catch(function (e) {
                        console.log(e);
                    })

            });
            rl.on("end", function(){
            });
    }).catch(function (e) {
        console.log(e);
    });
console output:
readed : 1000
readed : 2000
inserted to db : 1000
inserted to db : 2000
readed : 3000
readed : 4000
inserted to db : 3000
inserted to db : 4000
readed : 5000
readed : 6000
inserted to db : 5000
inserted to db : 6000
readed : 7000
readed : 8000
inserted to db : 7000
inserted to db : 8000
readed : 9000
readed : 10000
inserted to db : 9000
inserted to db : 10000
readed : 11000
readed : 12000
inserted to db : 11000
inserted to db : 12000
readed : 13000
inserted to db : 13000
readed : 14000
readed : 15000
inserted to db : 14000
inserted to db : 15000
readed : 16000
readed : 17000
inserted to db : 16000
Error
    at TarantoolConnection._processResponse (/home/akudelka/home/tam.by/ad.tam.by/server/node_modules/tarantool-driver/lib/connection.js:680:18)
    at TarantoolConnection.onData (/home/akudelka/home/tam.by/ad.tam.by/server/node_modules/tarantool-driver/lib/connection.js:406:22)
    at emitOne (events.js:96:13)
    at Socket.emit (events.js:191:7)
    at readableAddChunk (_stream_readable.js:178:18)
    at Socket.Readable.push (_stream_readable.js:136:10)
    at TCP.onread (net.js:561:20)
readed : 18000
readed : 19000
readed : 20000
readed : 21000
readed : 22000

В файле более 5 млн записей
Короче проблема в том что видимо закрывается соединение и на этом все. Есть ли способ решить данную проблему? Заранее спасибо.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions