*** Settings ***
Library           Resources/keywords/base_operate.py
Library           Resources/test/curve_base_test.py
Library           Resources/swig/swig_operate.py
Library			  Collections
#Library           RequestsLibrary
Library           Process
#Library           SSHLibrary
Library           Resources/keywords/mythread.py
Library           Resources/keywords/fault_inject.py
Library           Resources/keywords/deploy.py
Library           Resources/keywords/test_curve_stability_nbd.py

Suite Setup       init failover cluster
Suite Teardown    clean failover env

*** Variables ***
${fio_vdsize}     10
${vdbench_vdsize}   10
${vm_iops_limit}   8000
${chunk_num}   0

*** Test Cases ***
#test up all cs
#    [Tags]    P8    base    first release
#    up all cs
#    [Teardown]   file clean    none
   
inject one chunkserver down/up
    [Tags]    P1    base    first release   failover
    ${num}   evaluate   int(1)
    ${host}   test kill chunkserver num   ${num}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    sleep  5
    test start chunkserver num   ${num}  ${host}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	  start host cs process   ${host}

inject two chunkserver down/up
    [Tags]    P1    base    first release   failover
    ${num}   evaluate   int(2)
    ${host}   test kill chunkserver num   ${num}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    sleep  5
    test start chunkserver num   ${num}  ${host}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	  start host cs process   ${host}

inject host all chunkserver down/up
    [Tags]    P1    base    first release   failover
    ${host}  test stop chunkserver host
    sleep  10
    check nbd iops
    check data consistency
#    check copies consistency
    sleep  5
    test start chunkserver host  ${host}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	  start host cs process  ${host}

inject restart one chunkserver
    [Tags]    P1    base    first release   failover
    ${num}   evaluate   int(1)
    test restart chunkserver num   ${num}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	  wait cluster healthy   ${vm_iops_limit}

inject restart two chunkserver
    [Tags]    P1    base    first release   failover
    ${num}   evaluate   int(2)
    test restart chunkserver num   ${num}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	 wait cluster healthy    ${vm_iops_limit}

inject reboot nebd
    [Tags]   P1  base   first release   failover
    test reboot nebd
    sleep   5
    check nbd iops
    check data consistency
    [Teardown]  wait cluster healthy   ${vm_iops_limit}

inject kill one mds 
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(1)
    test kill mds   ${num}
    check nbd iops  
    sleep  3
    test start mds
    check nbd iops  
    check data consistency
    [Teardown]	  test start mds   

inject kill two mds
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(2)
    test kill mds   ${num}
    check nbd iops  
    sleep  3
    test start mds
    sleep  3
    check nbd iops  
    check data consistency
    [Teardown]    test start mds

inject round restart mds
    [Tags]    P1    base    first release   failover
    test round restart mds
    sleep  3
    check nbd iops  
    check data consistency
    [Teardown]    test start mds

inject loop restart one mds
    [Tags]    P1    base    first release   failover
    ${mds_ip}    evaluate    str("10.182.26.25")
    :FOR   ${i}   IN RANGE   10
          kill mds process   ${mds_ip}
          start mds process  ${mds_ip}
          sleep  3
          check nbd iops  
    END 
    check data consistency
    [Teardown]    test start mds


inject kill one etcd     
    [Tags]    P1    base    first release   failover   
    ${host}  test kill etcd
    sleep  5
    check nbd iops
    sleep  3
    test start etcd 
    test start mds
    test start snap
#    sleep  3
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]     start etcd process  ${host}

test round restart etcd
    [Tags]    P1    base    first release   failover   
    test round restart etcd
    sleep  3
    check nbd iops
    check data consistency
    test start mds
    test start snap
    [Teardown]    test start etcd

inject loop restart one etcd
    [Tags]    P1    base    first release   failover  
    ${etcd_ip}    evaluate    str("10.182.26.16")
    :FOR   ${i}   IN RANGE   10
          kill etcd process   ${etcd_ip}
          start etcd process  ${etcd_ip}
          sleep  2
          check nbd iops
    END
    check data consistency
    test start mds
    test start snap
    [Teardown]    test start etcd

inject chunkserver cpu stress 95%
    [Tags]    P1    base    first release   failover 
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test chunkserver cpu stress   ${cpu_stress}
    sleep  5
    ${limit_iops}   evaluate  int(2000)
    check nbd iops    ${limit_iops} 
    del cpu stress   ${ssh}
    check nbd iops    ${vm_iops_limit} 
    check data consistency
#    check copies consistency
    [Teardown]     del cpu stress    ${ssh}
    
inject mds cpu stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test mds cpu stress   ${cpu_stress}
    sleep  5
    ${limit_iops}   evaluate  int(2000)
    check nbd iops    ${limit_iops} 
    del cpu stress   ${ssh}
    check nbd iops    ${vm_iops_limit} 
    check data consistency
#    check copies consistency
    [Teardown]     del cpu stress    ${ssh}

inject client cpu stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test client cpu stress   ${cpu_stress}
    sleep  5
    ${limit_iops}   evaluate  int(2000)
    check nbd iops    ${limit_iops} 
    del cpu stress   ${ssh}
    check nbd iops    ${vm_iops_limit} 
    check data consistency
#    check copies consistency
    [Teardown]     del cpu stress    ${ssh}

inject chunkserver mem stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test chunkserver mem stress   ${cpu_stress}
    sleep  10
    check nbd iops    ${vm_iops_limit}
    del mem stress   ${ssh}
    check data consistency
#    check copies consistency
    [Teardown]     del mem stress    ${ssh}

inject mds mem stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test mds mem stress   ${cpu_stress}
    sleep  10
    check nbd iops    ${vm_iops_limit}
    del mem stress   ${ssh}
    check data consistency
#    check copies consistency
    [Teardown]     del mem stress    ${ssh}

inject client mem stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test client mem stress   ${cpu_stress}
    sleep  10
    check nbd iops    ${vm_iops_limit}
    del mem stress   ${ssh}
    check data consistency
#    check copies consistency
    [Teardown]     del mem stress    ${ssh}

test chunkserver network stress
    [Tags]    P1    base    first release   failover
    ${ip}    test chunkserver network stress
    sleep  10
    check nbd iops    ${vm_iops_limit}
    check data consistency
#    stop network stress    ${ip}    
    [Teardown]     stop network stress    ${ip}

test mds network stress
    [Tags]    P1    base    first release   failover
    ${ip}    test mds network stress
    sleep  10
    check nbd iops    ${vm_iops_limit}
    check data consistency
#    stop network stress    ${ip}
    [Teardown]     stop network stress    ${ip}

test client network stress
    [Tags]    P1    base    first release   failover
    ${ip}    test client network stress
    sleep  10
    check nbd iops    ${vm_iops_limit}
    check data consistency
#   stop network stress    ${ip}
    [Teardown]     stop network stress    ${ip}

test chunkserver clock offset 2min
    [Tags]    P1    base    first release   failover   no-need
    ${offset}  evaluate  int(2)
    ${ssh}    test chunkserver clock offset    ${offset}
    check nbd iops   ${vm_iops_limit}
    check data consistency
    [Teardown]   del clock offset   ${ssh}    ${offset}

test mds clock offset 2min
    [Tags]    P1    base    first release   failover   no-need
    ${offset}  evaluate  int(2)
    ${ssh}    test mds clock offset    ${offset}
    check nbd iops   ${vm_iops_limit}
    check data consistency
#   check copies consistency
    [Teardown]   del clock offset   ${ssh}    ${offset}

test chunkserver loss package 5%
    [Tags]    P1    base    first release   failover
    ${percent}  evaluate  int(5)
    test cs loss package  ${percent}
    check nbd iops
    test start mds
    test start snap
    check data consistency
#    check copies consistency
#    [Teardown]	   wait cluster healthy    ${vm_iops_limit}

test chunkserver delay package 50ms
    [Tags]    P1    base    first release   failover   
    ${ms}  evaluate  int(50)
    test cs delay package  ${ms}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    test start mds
    test start snap
    sleep   5
    [Teardown]	 wait cluster healthy    ${vm_iops_limit}


test out/in chunkserver recover
    [Tags]    P1    base    first release   failover  longtime
    ${host}	${cs_copyset_num}  test outcs recover copyset
    check nbd iops
    check data consistency
#    check copies consistency
    ${copyset_num}       Convert To Integer    ${cs_copyset_num}
    test upcs recover copyset  ${host}	 ${copyset_num}
    sleep  5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	   wait cluster healthy    ${vm_iops_limit}

test suspend delete and up recover chunkserver
    [Tags]    P1    base    first release   failover  longtime  no-need
    test suspend delete recover copyset
#    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	   wait cluster healthy   ${vm_iops_limit}

test stop all cs not recover
    [Tags]    P1    base    first release   failover
    stop all cs not recover
    sleep   5
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	  wait cluster healthy   ${vm_iops_limit}

test pendding host all cs
   [Tags]    P1    base    first release   failover  longtime
   pendding all cs recover 
   sleep  5
   check nbd iops 
   check data consistency
#   check copies consistency
   [Teardown]	  check chunkserver online

test mds loss package 5%   
    [Tags]    P1    base    first release   failover  
    ${percent}  evaluate  int(5)
    test mds loss package  ${percent}
    sleep  60
    check nbd iops
    check data consistency
#    check copies consistency
    test start mds
    test start snap
    [Teardown]	   wait cluster healthy   ${vm_iops_limit}

test ipmitool restart mds node
   [Tags]    P1    base    first release   failover  ipmitool
   test ipmitool restart mds
   sleep  60
   check nbd iops
   check data consistency
   test start mds
   test start snap
   [Teardown]    wait cluster healthy   ${vm_iops_limit}

test ipmitool restart chunkserver node
   [Tags]    P1    base    first release   failover  ipmitool
   test ipmitool restart chunkserver
   sleep  60
   check nbd iops
   check data consistency
   [Teardown]    check chunkserver online

stop map unmap thread
    [Tags]    P1    base    first release   failover
    stop map unmap
#    [Teardown]   wait cluster healthy    ${vm_iops_limit}

stop snapshot test
    [Tags]    P1    base    first release   failover  
    stop snapshot test

test lazy clone fail flatten and snapshot
    [Tags]    P1    base    first release   failover   
    test lazy clone flatten snapshot fail 
    
test mds delay package 50ms
    [Tags]    P1    base    first release   failover   
    ${ms}  evaluate  int(50)
    test mds delay package  ${ms}
    test start mds
    test start etcd
    test start snap
    sleep  120
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	   wait cluster healthy   ${vm_iops_limit}

inject kill diff host chunkserver
    [Tags]    P1    base    first release   failover
    test kill diff host chunkserver
    wait health ok
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]	 wait cluster healthy    ${vm_iops_limit}

inject kill all mds
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(3)
    test kill mds   ${num}
    test start mds
    sleep  3
    check nbd iops  
    check data consistency
    [Teardown]    test start mds

inject kill two etcd
    [Tags]    P1    base    first release   failover  
    ${num}   evaluate  int(2)
    test kill etcd   ${num}
    sleep  5
#    check nbd iops
    sleep  3
    test start etcd
    test start mds
    test start snap
    sleep  3
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]     test start etcd

inject all chunkserver down
    [Tags]    P1    base    first release   failover
    test stop all chunkserver
    test start all chunkserver
    wait health ok
    sleep  10
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]   wait cluster healthy    ${vm_iops_limit}

inject all chunkserver down 10min
    [Tags]    P1    base    first release   failover  
    test stop all chunkserver
    sleep   600
    test start all chunkserver
    wait health ok
    sleep  10
    check nbd iops
    check data consistency
#    check copies consistency
    [Teardown]   wait cluster healthy    ${vm_iops_limit}

test clone vol
    [Tags]    P1    base    first release   failover  snapshot  no-need
    test clone iovol consistency

test cancel vol snapshot
    [Tags]    P1    base    first release   failover   snapshot  no-need
    test cancel snapshot

test one volume perf
    [Tags]    P1    base    first release  longtime   perf  no-need
    remove vm key
    attach new vol   200   200
    stop rwio
    perf test

inject cluster stress test
    [Tags]    P1    base    first release  longtime  stress   perf  no-need
    init vm
    remove vm key
    attach new vol   20   20
    stop rwio
    stress test

*** Keywords ***

init failover cluster
     clean nbd  
     destroy test env 
     ${begin_num}    get all chunk num
     ${begin_num}    Convert To Integer   ${begin_num}
     Set Global Variable  ${chunk_num}    ${begin_num}
     init abnormal test env
     check chunkserver online
     init rw cloud disk
     begin snapshot test
     begin kill snapshotserver
     loop map unmap file

init abnormal test env
     clean env
#     destroy etcd
#     destroy mds
#     destroy snapshotclone server
#     stop nebd
#     drop all chunkserver dat
     use ansible deploy
#     install deb
#     add config 
#     start abnormal test services
#     create pool
#     start nebd

init rw cloud disk
     map nbd
     stop rwio
     write full disk  ${fio_vdsize}
     run rwio
     clean kernel log

clean failover env
     stop rwio
     check io error
     stop scheduler
     check copies consistency
     clean nbd
     delete nbd
     sleep  10
     clear RecycleBin
     recycle chunk
     sleep  60 
     ${final_num}    get all chunk num
     ${final_num}    Convert To Integer   ${final_num}
     should be equal    ${final_num}   ${chunk_num}
