模拟大量的并发读场景，storaged监测不到写IO，反而存在少量读IO，是什么原理呢

问答 问题
, , ,
1
  • nebula 版本：3.8.0
  • 部署方式：k8s helm包部署，3台机器，3个副本，3个分片
  • 安装方式：operator
  • 是否上生产环境：N
  • 硬件信息
    • 磁盘：HDD
    • CPU、内存信息：32C 256G
  • 问题的具体描述
    客户端发起大量的并发查询，在部署的机器上，查看storaged进程的IO占用情况，发现没有任何的读IO，反倒是有少量的写IO，是什么原理？
    image
    image1026×730 72.6 KB

    资源占用如图：
    image
    image820×599 35 KB

客户端的代码如下：

func main() {
    for {
       Range(200)
    }

    pool.Close()
}

func Range(num int) time.Duration {
    wg := sync.WaitGroup{}
    sessions := make([]*nebula_go.Session, 0)
    //for i := 0; i < 5; i++ {
    // sessions = append(sessions, getSession())
    // //_, err := sessions[i].Execute(fmt.Sprintf(`USE test_wc_1_%d_tenant`, i+1))
    // _, err := sessions[i].Execute(fmt.Sprintf(`USE topo_85761044_21`))
    // //_, err := sessions[i].Execute(fmt.Sprintf(`USE test_wc_6_2_tenant`))
    // if err != nil {
    //    fmt.Println("err:", err)
    // }
    //}
    for i := 0; i < num; i++ {
       sessions = append(sessions, getSession())
       _, err := sessions[i].Execute(fmt.Sprintf(`USE test_wc_14_%d_tenant`, (i/5)+1))
       //_, err := sessions[i].Execute(fmt.Sprintf(`USE test_wc_14_%d_tenant`, (i/25)+1))
       //_, err := sessions[i].Execute(fmt.Sprintf(`USE topo_85761044_2`))
       if err != nil {
          fmt.Println("err:", err)
       }
    }
    idStr := `
    "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-w0962d2qwd", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-c4r1rsdzgw", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-d7n4ac8mpc", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-vlwy2qyex2", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-zqclm15rvx", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-825zlb20g5", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-0jeeu5s7rw", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-wy9ij10x0t", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-d505nj4ar5", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-is2ue8zyxm", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-sper2fo9mg", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-amcfwzjxci", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-hr6vlqst97", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-b0p1nnd1pk", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-x4n0pj7076", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-42pd6l2w27", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-bup3yj7hix", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-erfl5fkakd", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-xeqzab5wcz", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-04ndffshjx"
`
    idList := `["bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-w0962d2qwd", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-c4r1rsdzgw", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-d7n4ac8mpc", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-vlwy2qyex2", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-zqclm15rvx", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-825zlb20g5", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-0jeeu5s7rw", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-wy9ij10x0t", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-d505nj4ar5", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-is2ue8zyxm", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-sper2fo9mg", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-amcfwzjxci", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-hr6vlqst97", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-b0p1nnd1pk", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-x4n0pj7076", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-42pd6l2w27", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-bup3yj7hix", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-erfl5fkakd", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-xeqzab5wcz", "bebb0f90-072a-4e1c-ac77-850d6741e029:subnet-04ndffshjx"]`
    wg.Add(len(sessions))
    idStr2 := `"bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-zdz1wrm41v", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-wupk8dpokq", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-szcpzen1ib", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-a8rppy1p9h", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-4pb9v81071", "bebb0f90-072a-4e1c-ac77-850d6741e029:db-gmvf8vs978", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-oakvzp661a", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-k9dx5359e4", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-gjk4me6mfp", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9jwobca326", "bebb0f90-072a-4e1c-ac77-850d6741e029:db-vzlj0fo8c4", "bebb0f90-072a-4e1c-ac77-850d6741e029:db-7js8j47jf8", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-z0v9icf4rk", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-ylej37brr2", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-rm1uq0a7ag", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-rh9s8f21ni", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-mudgms088a", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-k3tlznkdx5", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-hpkgp3xbgk", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-c1kxu3wo2z", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-62inimothe", "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-3o8tfcdb6c", "bebb0f90-072a-4e1c-ac77-850d6741e029:db-uxzqy4idx7", "bebb0f90-072a-4e1c-ac77-850d6741e029:db-9s9tjwe51u"`
    var total time.Duration
    idx := rand.Intn(len(sessions))
    for i := 0; i < len(sessions); i++ {
       tmp := i
       go func() {
          start := time.Now()
          defer func() {
             wg.Done()
             if tmp == idx {
                cost := time.Since(start)
                total = cost
                fmt.Printf("execute costs: %s\n", cost.String())
             }
          }()
          resp, err := sessions[tmp].Execute(fmt.Sprintf(`MATCH (account:cloud_account)-[e:accessible]-()RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          if resp.GetErrorCode() != nebula_go.ErrorCode_SUCCEEDED {
             fmt.Println("err msg:", resp.GetErrorMsg())
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (region:custom_region)-[e:accessible]-(:internet) RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (:custom_region)-[e:accessible]-(:custom_region) RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v : vpc ) -[ e : accessible ]-()
 WHERE v. vpc ["cloud_account_id"]=="bebb0f90-072a-4e1c-ac77-850d6741e029"
RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`FIND NOLOOP PATH WITH PROP FROM %s TO "internet" OVER net2sbnt UPTO 3 STEPS
YIELD PATH AS p`, idStr))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`FIND NOLOOP PATH WITH PROP FROM "internet" TO %s OVER net2sbnt UPTO 3 STEPS
YIELD PATH AS p`, idStr))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p= ( v : subnet ) -[ : net2sbnt ]->( : fw_nat ) -[ : net2sbnt ]->( : nat ) -[ : net2sbnt ]->( : fw_internet ) -[ : net2sbnt ]->( : internet )
 WHERE id(v) IN %s
RETURN p`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p= ( : internet ) -[ : net2sbnt ]->( : fw_internet ) -[ : net2sbnt ]->( : nat ) -[ : net2sbnt ]->( : fw_nat ) -[ : net2sbnt ]->( v : subnet )
 WHERE id(v) IN %s
RETURN p`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p= ( : internet ) -[ : net2sbnt ]->( : fw_internet ) -[ : net2sbnt ]->( : waf )
-[ : net2sbnt ]->( t1 : transfer )-[ : net2sbnt ]->( v : subnet )
WHERE id(v) IN %s AND any(t IN ["lb","nat","eip"] WHERE t IN LABELS(t1)) RETURN p`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p= ( : internet ) -[ : net2sbnt ]->( t1 : transfer ) -[ : net2sbnt*0..1 ]->( : waf )
-[ : net2sbnt ]->(  : nat ) -[ : net2sbnt ]->(  : fw_nat )-[ : net2sbnt ]->( v : subnet )
WHERE id(v) IN %s AND any(t IN ["fw_internet","waf"] WHERE t IN LABELS(t1)) RETURN p`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : subnet ) -[ : rg2sbnt ]->( v2 : transfer ) -[ : rg2sbnt ]->( v3 : region )
 WHERE id(v1) IN %s
RETURN v1, v2, v3`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : region ) -[ : rg2sbnt ]->( v2 : transfer ) -[ : rg2sbnt ]->( v3 : subnet )
 WHERE id(v3) IN %s
RETURN v1, v2, v3`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : subnet ) -[ : crsvpc ]->( v2 : transfer ) -[ : crsvpc ]->( v3 : vpc )
 WHERE id(v1) IN %s AND id(v3)!="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN v1, v2, v3`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : vpc ) -[ : crsvpc ]->( v2 : transfer ) -[ : crsvpc ]->( v3 : subnet )
 WHERE id(v3) IN %s AND id(v1)!="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN v1, v2, v3`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : subnet ) -[ : cen2vpc ]->( v2 : transfer ) -[ : cen2vpc ]->( v3 : transfer ) -[ : cen2vpc ]->( v4 : vpc )
 WHERE id(v1) IN %s AND id(v4)!="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN v1, v2, v3, v4`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : vpc ) -[ : cen2vpc ]->( v2 : transfer ) -[ : cen2vpc ]->( v3 : transfer ) -[ : cen2vpc ]->( v4 : subnet )
 WHERE id(v4) IN %s AND id(v1)!="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN v1, v2, v3, v4`, idList))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`GO FROM %s OVER forward BIDIRECT
 YIELD EDGE AS e`, idStr2))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`GO FROM "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-oakvzp661a" OVER forward BIDIRECT
 YIELD EDGE AS e, '' AS subnetId, '' AS subnet0Id
UNION
GO FROM "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-oakvzp661a" OVER accessible BIDIRECT
 WHERE properties($^). vpcId == properties($$). vpcId
 YIELD EDGE AS e, properties($$).subnetId AS subnetId, properties($^).subnetId AS subnet0Id`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (target:endpoint)-[e:accessible]->(e1:endpoint)
       WHERE id(e1)== "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-k9dx5359e4" AND 'internet' NOT  IN tags(target) AND  'custom_region' NOT  IN tags(target)
       return DISTINCT count(target) as t`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (target:endpoint)-[e:accessible]->(e1:endpoint)
       WHERE id(e1) == "bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-a8rppy1p9h"
       AND 'internet' NOT  IN tags(target) AND  'custom_region' NOT  IN tags(target)
       WITH id(target) as assetId ORDER BY assetId  DESC SKIP 0 LIMIT 2000 RETURN assetId`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (account:cloud_account)-[e:accessible]-() WHERE id(account) IN ["74501dfa-586f-4b6f-86a5-c62bdbe5b8b0", "9e982b44-5a34-4f51-9b92-2371cccbc777", "9e90147c-476a-4229-ba4d-006592a4a966", "bebb0f90-072a-4e1c-ac77-850d6741e029"]
 RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : vpc ) -[ e : accessible ]-( v2 : vpc )
 WHERE v1. vpc ["cloud_account_id"] IN ["74501dfa-586f-4b6f-86a5-c62bdbe5b8b0", "9e982b44-5a34-4f51-9b92-2371cccbc777", "9e90147c-476a-4229-ba4d-006592a4a966", "bebb0f90-072a-4e1c-ac77-850d6741e029"]
 AND v1. vpc ["cloud_account_id"] != v2. vpc ["cloud_account_id"]
RETURN DISTINCT e`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (v:subnet) return v`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (v:vpc) return v`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (v:peer) return v`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (a:endpoint) RETURN count(a) AS t`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (a:endpoint) WITH a, id(a) AS vid ORDER BY vid DESC SKIP 0 LIMIT 100000 return a as s1`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p=(a:internet)-[b:accessible]->(c:endpoint) RETURN c AS s1`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`FIND NOLOOP PATH WITH PROP FROM 'internet' TO 'bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5' OVER forward UPTO 3 STEPS YIELD PATH AS p`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p=(:internet)-[:forward]->(:fw_internet)-[:forward]->(:waf)-[:forward]->(t1:transfer)-[:forward]->(v:endpoint)
WHERE id(v)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5" AND any(t IN ["lb","nat","eip"] WHERE t IN LABELS(t1)) RETURN p`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p=(:internet)-[:forward]->(t1:fw_internet)-[:forward]->(:nat)-[:forward]->(:fw_nat)-[:forward]->(v:endpoint)
WHERE id(v)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5" RETURN p`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p=(:internet)-[:forward]->(:waf)-[:forward]->(:nat)-[:forward]->(:fw_nat)-[:forward]->(v:endpoint) WHERE id(v)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5" RETURN p`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH p=(:internet)-[:forward]->(:fw_internet)-[:forward]->(:waf)-[:forward]->(:nat)-[:forward]->(:fw_nat)-[:forward]->(v:endpoint) WHERE id(v)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-9vmrkddqz5" RETURN p`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : endpoint ) <-[ : accessible ]-( v2 : endpoint )
 WHERE id(v1)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-mudgms088a" AND properties(v2). vpcId == properties(v1). vpcId
RETURN v2`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH (v) <-[ : accessible ]-( vpcs : vpc )
 WHERE id(v)=="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN vpcs`))
          if err != nil {
             fmt.Println("err:", err)
          }
          _, err = sessions[tmp].Execute(fmt.Sprintf(`MATCH ( v1 : endpoint ) <-[ : accessible ]-( v2 : endpoint )
 WHERE id(v1)=="bebb0f90-072a-4e1c-ac77-850d6741e029:ecs-mudgms088a" AND properties(v2). vpcId =="bebb0f90-072a-4e1c-ac77-850d6741e029:vpc-fxg8wlrl04"
RETURN v2`))
          if err != nil {
             fmt.Println("err:", err)
          }
          //_, err = sessions[tmp].Execute(fmt.Sprintf(``))
          //if err != nil {
          // fmt.Println("err:", err)
          //}
          //_, err = sessions[tmp].Execute(fmt.Sprintf(``))
          //if err != nil {
          // fmt.Println("err:", err)
          //}
       }()
    }
    wg.Wait()
    for _, session := range sessions {
       session.Release()
    }
    time.Sleep(time.Second * 3)
    return total
}

想了下，这种情况，如果没有写入IO，可能是进行了数据预热？因为每次查询的内容和结果都一模一样，结果直接从内存中的缓存返回，但是从查询结果返回给客户端的时间来看，这个假设貌似不成立，因为上述查询，返回都非常慢，且资源消耗非常高，也不像是直接命中缓存什么的

希望官方大佬可以出来解答下小白的2个疑问：

  1. 为什么只有查询，没有读IO，反而有写IO？
  2. 为什么没有读IO，查询消耗的资源这么高？资源消耗用在了哪个地方？

谢谢！

2

mark 下。
怀疑你有写 IO 是其他原因导致，不然这个非常不科学。
没有读 IO 怀疑是缓存到 storage 了。资源消耗在并发查询的处理和计算上

3

写IO应当就是客户端查询导致的，我将客户端停掉，读写IO就都没了，客户端一启动，写IO就有了，但读IO始终为0