The framework [testify](https://github.com/stretchr/testify/tree/v1.8.4) might be the most popular. Surprisingly, I find the documentation is not clear enough to answer my questions and I can't find answers anywhere. Therefore, I have to write experiments to verify my guesses.
One of my questions is: when will the registered Cleanup functions run?
The summary of my experiment shows that the order of Cleanup functions is running as we normally expected except for those from subtests. The exception could lead severe problems.
The experiment
The idea is simple: register Cleanup functions from all setup/teardown methods and use the log to infer the execution order.
Let's have a peek at the result first.
We can see that the Cleanup functions registered at TearDownSubTest and SetupSubTest are called after the subtests are finished. However, it's NOT guaranteed that they will be called immediately after a subtest is finished.
There are two scenarios:
- For nested subtests, the
Cleanupfunctions registered atTearDownSubTestandSetupSubTestare called once their parents are finished but not cleaned. - For direct subtests whose parents are NOT subtests, their
Cleanupfunctions registered atTearDownSubTestandSetupSubTestare called once the parents'AfterTestandTearDownTestare finished.
The impact is:
- Nested subtests could pollute each other because some
Cleanupfunctions are running after all sibling subtests. - Children subtest can even pollute their parents.
The source code can be found here: https://github.com/xuanyuwang/testify-examples/blob/main/cleanup-order/cleanup_order_test.go
package cleanuporder_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/suite"
)
var tearDownSubTest []string
type TestCleanupOrder struct {
suite.Suite
}
func (s *TestCleanupOrder) SetupSuite() {
fmt.Println("Run Interface SetupAllSuite: SetupSuite")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface SetupAllSuite: SetupSuite")
})
}
func (s *TestCleanupOrder) SetupTest() {
fmt.Println("Run Interface SetupTestSuite: SetupTest")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface SetupTestSuite: SetupTest")
})
}
func (s *TestCleanupOrder) TeatDownSuite() {
fmt.Println("Run Interface TearDownAllSuite: TearDownSuite")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface TearDownAllSuite: TearDownSuite")
})
}
func (s *TestCleanupOrder) TearDownTest() {
fmt.Println("Run Interface TearDownTestSuite: TearDownTest")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface TearDownTestSuite: TearDownTest")
})
}
func (s *TestCleanupOrder) BeforeTest(suiteName, testName string) {
fmt.Printf("Run Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
s.T().Cleanup(func() {
fmt.Printf("Cleanup Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
})
}
func (s *TestCleanupOrder) AfterTest(suiteName, testName string) {
fmt.Printf("Run Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
s.T().Cleanup(func() {
fmt.Printf("Cleanup Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
})
}
func (s *TestCleanupOrder) HandleStats(suiteName string, stats *suite.SuiteInformation) {
fmt.Printf("Run Interface WithStats: HandleStats for suite %s\n", suiteName)
s.T().Cleanup(func() {
fmt.Printf("Cleanup Interface WithStats: HandleStats for suite %s\n", suiteName)
})
}
func (s *TestCleanupOrder) SetupSubTest() {
fmt.Println("Run Interface SetupSubTest: SetupSubTest")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface SetupSubTest: SetupSubTest")
})
}
func (s *TestCleanupOrder) TearDownSubTest() {
fmt.Println("Run Interface TearDownSubTest: TearDownSubTest")
s.T().Cleanup(func() {
fmt.Println("Cleanup Interface TearDownSubTest: TearDownSubTest")
fmt.Printf("\tTear down sub test %v\n", tearDownSubTest)
})
}
// Test case
func (s *TestCleanupOrder) TestCaseA() {
fmt.Println("Run TestCase A")
s.T().Cleanup(func() {
fmt.Println("Cleanup TestCase A")
})
s.Run("Sub-TestCase A", func() {
fmt.Println("Run Sub-TestCase A")
s.T().Cleanup(func() {
fmt.Println("Cleanup Sub-TestCase A")
})
tearDownSubTest = append(tearDownSubTest, "Sub-TestCase A")
s.Run("Sub-Sub-TestCase A", func() {
s.T().Cleanup(func() {
fmt.Println("Cleanup Sub-Sub-TestCase A")
})
fmt.Println("Run Sub-Sub-TestCase A")
tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase A")
fmt.Println("Finish Sub-Sub-TestCase A")
})
s.Run("Sub-Sub-TestCase B", func() {
s.T().Cleanup(func() {
fmt.Println("Cleanup Sub-Sub-TestCase B")
})
fmt.Println("Run Sub-Sub-TestCase B")
tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase B")
fmt.Println("Finish Sub-Sub-TestCase B")
})
fmt.Println("Finish Sub-TestCase A")
})
fmt.Println("Finish TestCase A")
}
// Entry point of the whole test suite
func TestCleanupOrderSuite(t *testing.T) {
suite.Run(t, &TestCleanupOrder{})
}
Top comments (0)
Some comments may only be visible to logged-in visitors. Sign in to view all comments.