Nebula python session 第二次执行 query 时 blocked，并抛出 Socket read failed: [Errno 60] Operation timed out

FrankieYao · 2023 年10 月 29 日 03:53

版本信息：
Python: 3.11.0
Nebula: 3.6.0
nebula3-python: 3.4.0
异常信息：

➜  neo4j-benchmark git:(main) ✗ /usr/local/bin/python3 /Users/yicyao/python-projects/neo4j-benchmark/nebula_client.py
ResultSet(...); cost time: 274.40810203552246
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/TSocket.py", line 305, in read
    buff = self.handle.recv(sz)
           ^^^^^^^^^^^^^^^^^^^^
TimeoutError: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/gclient/net/Connection.py", line 148, in execute_parameter
    resp = self._connection.executeWithParameter(session_id, stmt, params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/graph/GraphService.py", line 1652, in executeWithParameter
    return self.recv_executeWithParameter()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/graph/GraphService.py", line 1665, in recv_executeWithParameter
    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/protocol/THeaderProtocol.py", line 157, in readMessageBegin
    self.trans._reset_protocol()
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/THeaderTransport.py", line 238, in _reset_protocol
    self.readFrame(0)
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/THeaderTransport.py", line 268, in readFrame
    word1 = self.getTransport().readAll(4)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/TTransport.py", line 72, in readAll
    chunk = self.read(need)
            ^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/TTransport.py", line 183, in read
    self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size)))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/fbthrift/transport/TSocket.py", line 310, in read
    raise TTransportException(
nebula3.fbthrift.transport.TTransport.TTransportException: Socket read failed: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/yicyao/python-projects/neo4j-benchmark/nebula_client.py", line 25, in <module>
    execute_query(query)
  File "/Users/yicyao/python-projects/neo4j-benchmark/nebula_client.py", line 17, in execute_query
    result = session.execute(query)
             ^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/gclient/net/Session.py", line 77, in execute
    return self.execute_parameter(stmt, None)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/gclient/net/Session.py", line 42, in execute_parameter
    resp = self._connection.execute_parameter(self._session_id, stmt, params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/nebula3/gclient/net/Connection.py", line 154, in execute_parameter
    raise IOErrorException(IOErrorException.E_TIMEOUT, te.message)
nebula3.Exception.IOErrorException: Socket read failed: timed out
Closing a connection that is in use

代码详情

from nebula3.Config import Config
from nebula3.gclient.net.SessionPool import SessionPool
from nebula3.gclient.net import ConnectionPool
import time
import numpy as np

config = Config()
config.timeout = 100000
config.max_connection_pool_size = 10
connection_pool = ConnectionPool()
ok = connection_pool.init([('...', 9669)], config)
session = connection_pool.get_session('root', 'nebula')
session.execute('USE dataset_1')

def execute_query(query):
    start_time = time.time()
    result = session.execute(query)
    print(result)
    end_time = time.time()
    print("{} cost time: {}".format(query, float(end_time - start_time) * 1000.0))

size = 2
for i in range(size):
    query = "..."
    execute_query(query)

其他信息：

nebula 通过 docker-compose 部署在一台服务器上面，默认配置。

FrankieYao · 2023 年10 月 29 日 03:59

我也尝试了 sessionPool，遇到一样的问题，第一次 200+ ms 就返回了，第二次就超时

from nebula3.gclient.net.SessionPool import SessionPool
from nebula3.gclient.net import Connection
from nebula3.Config import SessionPoolConfig
from nebula3.common import *
import time

ip = '...'
port = 9669

config = SessionPoolConfig()
conn = Connection()
conn.open(ip, port, 1000)
auth_result = conn.authenticate('root', 'nebula')
assert auth_result.get_session_id() != 0

session_pool = SessionPool('root', 'nebula', 'dataset_1', [(ip, port)])
session_pool.init(config)


def execute_query(query):
    start_time = time.time()
    result = session_pool.execute(query)
    print(result)
    end_time = time.time()
    cost_time = float(end_time - start_time) * 1000.0
    print("{} cost time: {}".format(query, cost_time))

for i in range(2):
    query = ".."
    execute_query(query)

FrankieYao · 2023 年10 月 30 日 01:30

我把 Nebula 的版本降到 3.4.0 还是遇到同样的问题。

FrankieYao · 2023 年10 月 30 日 01:32

我尝试了一下 go client，发现前面2次 query 正常，第3次被 block

package main

import (
	"fmt"
	"math"
	"math/rand"
	"sort"
	"time"

	nebula "github.com/vesoft-inc/nebula-go/v3"
)

const (
	address  = "..."
	port     = 9669
	username = "root"
	password = "nebula"
	useHTTP2 = false
)

var log = nebula.DefaultLogger{}
var cost_time_list []float64

func computeStats(numbers []float64) {
	maxVal := math.SmallestNonzeroFloat64
	minVal := math.MaxFloat64
	sum := 0.0

	for _, num := range numbers {
		if num > maxVal {
			maxVal = num
		}
		if num < minVal {
			minVal = num
		}
		sum += num
	}

	average := sum / float64(len(numbers))

	sort.Float64s(numbers)
	p99index := int(0.99 * float64(len(numbers)))
	p95index := int(0.95 * float64(len(numbers)))
	p99 := numbers[p99index]
	p95 := numbers[p95index]

	fmt.Printf("Max: %.2f\n", maxVal)
	fmt.Printf("Min: %.2f\n", minVal)
	fmt.Printf("Average: %.2f\n", average)
	fmt.Printf("P99: %.2f\n", p99)
	fmt.Printf("P95: %.2f\n", p95)
}

func main() {
	hostAddress := nebula.HostAddress{Host: address, Port: port}
	hostList := []nebula.HostAddress{hostAddress}
	testPoolConfig := nebula.GetDefaultConf()
	testPoolConfig.UseHTTP2 = useHTTP2

	pool, err := nebula.NewConnectionPool(hostList, testPoolConfig, log)
	if err != nil {
		log.Fatal(fmt.Sprintf("Fail to initialize the connection pool, host: %s, port: %d, %s", address, port, err.Error()))
	}
	defer pool.Close()

	session, err := pool.GetSession(username, password)
	if err != nil {
		log.Fatal(fmt.Sprintf("Fail to create a new session from connection pool, username: %s, password: %s, %s",
			username, password, err.Error()))
	}
	defer session.Release()

	{
		i := 0
		for i < 10 {
			startTime := time.Now()
			sql := fmt.Sprintf("...")
			createSchema2 := "USE ...;" + sql
			resultSet2, err2 := session.Execute(createSchema2)
			fmt.Println(sql)
			fmt.Println(resultSet2.GetValuesByColName("..."))
			if err2 != nil {
				fmt.Print(err.Error())
				return
			}
			duration := time.Since(startTime)
			cost_time_list = append(cost_time_list, float64(duration.Milliseconds()))

			i++
		}
		computeStats(cost_time_list)
	}

	fmt.Print("\n")
	log.Info("Nebula Go Client Basic Example Finished")
}

wey · 2023 年11 月 7 日 07:35

看起来是连接到 graphd 会超时。

graphd 有没有崩溃重启呢？用 console 能重现么？

system · 2023 年12 月 7 日 07:36

此话题已在最后回复的 30 天后被自动关闭。不再允许新回复。