cluster_read_config.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. // Copyright 2018 The etcd Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package tester
  15. import (
  16. "errors"
  17. "fmt"
  18. "io/ioutil"
  19. "net/url"
  20. "path/filepath"
  21. "strings"
  22. "github.com/coreos/etcd/functional/rpcpb"
  23. "go.uber.org/zap"
  24. yaml "gopkg.in/yaml.v2"
  25. )
  26. func read(lg *zap.Logger, fpath string) (*Cluster, error) {
  27. bts, err := ioutil.ReadFile(fpath)
  28. if err != nil {
  29. return nil, err
  30. }
  31. lg.Info("opened configuration file", zap.String("path", fpath))
  32. clus := &Cluster{lg: lg}
  33. if err = yaml.Unmarshal(bts, clus); err != nil {
  34. return nil, err
  35. }
  36. if len(clus.Members) < 3 {
  37. return nil, fmt.Errorf("len(clus.Members) expects at least 3, got %d", len(clus.Members))
  38. }
  39. for i, mem := range clus.Members {
  40. if mem.BaseDir == "" {
  41. return nil, fmt.Errorf("BaseDir cannot be empty (got %q)", mem.BaseDir)
  42. }
  43. if mem.EtcdLogPath == "" {
  44. return nil, fmt.Errorf("EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath)
  45. }
  46. if mem.Etcd.Name == "" {
  47. return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem)
  48. }
  49. if mem.Etcd.DataDir == "" {
  50. return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem)
  51. }
  52. if mem.Etcd.SnapshotCount == 0 {
  53. return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount)
  54. }
  55. if mem.Etcd.DataDir == "" {
  56. return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir)
  57. }
  58. if mem.Etcd.WALDir == "" {
  59. clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal")
  60. }
  61. switch mem.Etcd.InitialClusterState {
  62. case "new":
  63. case "existing":
  64. default:
  65. return nil, fmt.Errorf("'--initial-cluster-state' got %q", mem.Etcd.InitialClusterState)
  66. }
  67. if mem.Etcd.HeartbeatIntervalMs == 0 {
  68. return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd)
  69. }
  70. if mem.Etcd.ElectionTimeoutMs == 0 {
  71. return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd)
  72. }
  73. if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs {
  74. return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs)
  75. }
  76. port := ""
  77. listenClientPorts := make([]string, len(clus.Members))
  78. for i, u := range mem.Etcd.ListenClientURLs {
  79. if !isValidURL(u) {
  80. return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u)
  81. }
  82. listenClientPorts[i], err = getPort(u)
  83. if err != nil {
  84. return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u)
  85. }
  86. }
  87. for i, u := range mem.Etcd.AdvertiseClientURLs {
  88. if !isValidURL(u) {
  89. return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u)
  90. }
  91. port, err = getPort(u)
  92. if err != nil {
  93. return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u)
  94. }
  95. if mem.EtcdClientProxy && listenClientPorts[i] == port {
  96. return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i])
  97. }
  98. }
  99. listenPeerPorts := make([]string, len(clus.Members))
  100. for i, u := range mem.Etcd.ListenPeerURLs {
  101. if !isValidURL(u) {
  102. return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u)
  103. }
  104. listenPeerPorts[i], err = getPort(u)
  105. if err != nil {
  106. return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u)
  107. }
  108. }
  109. for j, u := range mem.Etcd.AdvertisePeerURLs {
  110. if !isValidURL(u) {
  111. return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u)
  112. }
  113. port, err = getPort(u)
  114. if err != nil {
  115. return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u)
  116. }
  117. if mem.EtcdPeerProxy && listenPeerPorts[j] == port {
  118. return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j])
  119. }
  120. }
  121. if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) {
  122. return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath)
  123. }
  124. if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) {
  125. return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir)
  126. }
  127. // TODO: support separate WALDir that can be handled via failure-archive
  128. if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) {
  129. return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir)
  130. }
  131. // TODO: only support generated certs with TLS generator
  132. // deprecate auto TLS
  133. if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" {
  134. return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
  135. }
  136. if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" {
  137. return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile)
  138. }
  139. if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" {
  140. return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
  141. }
  142. if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" {
  143. return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile)
  144. }
  145. if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" {
  146. return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile)
  147. }
  148. if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" {
  149. return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
  150. }
  151. if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" {
  152. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
  153. }
  154. if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile == "" {
  155. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile)
  156. }
  157. // only support self-signed certs
  158. if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile == "" {
  159. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerCertFile)
  160. }
  161. if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" {
  162. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
  163. }
  164. if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile != "" {
  165. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile)
  166. }
  167. if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile != "" {
  168. return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
  169. }
  170. if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerAutoTLS {
  171. return nil, fmt.Errorf("Etcd.PeerClientCertAuth and Etcd.PeerAutoTLS cannot be both 'true'")
  172. }
  173. if (mem.Etcd.PeerCertFile == "") != (mem.Etcd.PeerKeyFile == "") {
  174. return nil, fmt.Errorf("Both Etcd.PeerCertFile %q and Etcd.PeerKeyFile %q must be either empty or non-empty", mem.Etcd.PeerCertFile, mem.Etcd.PeerKeyFile)
  175. }
  176. if mem.Etcd.ClientCertAuth && mem.Etcd.ClientAutoTLS {
  177. return nil, fmt.Errorf("Etcd.ClientCertAuth and Etcd.ClientAutoTLS cannot be both 'true'")
  178. }
  179. if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" {
  180. return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
  181. }
  182. if mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile == "" {
  183. return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile)
  184. }
  185. if mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile == "" {
  186. return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
  187. }
  188. if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" {
  189. return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
  190. }
  191. if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile != "" {
  192. return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile)
  193. }
  194. if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile != "" {
  195. return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.PeerCertFile)
  196. }
  197. if (mem.Etcd.ClientCertFile == "") != (mem.Etcd.ClientKeyFile == "") {
  198. return nil, fmt.Errorf("Both Etcd.ClientCertFile %q and Etcd.ClientKeyFile %q must be either empty or non-empty", mem.Etcd.ClientCertFile, mem.Etcd.ClientKeyFile)
  199. }
  200. peerTLS := mem.Etcd.PeerAutoTLS ||
  201. (mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" && mem.Etcd.PeerKeyFile != "" && mem.Etcd.PeerTrustedCAFile != "")
  202. if peerTLS {
  203. for _, cu := range mem.Etcd.ListenPeerURLs {
  204. var u *url.URL
  205. u, err = url.Parse(cu)
  206. if err != nil {
  207. return nil, err
  208. }
  209. if u.Scheme != "https" { // TODO: support unix
  210. return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
  211. }
  212. }
  213. for _, cu := range mem.Etcd.AdvertisePeerURLs {
  214. var u *url.URL
  215. u, err = url.Parse(cu)
  216. if err != nil {
  217. return nil, err
  218. }
  219. if u.Scheme != "https" { // TODO: support unix
  220. return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
  221. }
  222. }
  223. clus.Members[i].PeerCertPath = mem.Etcd.PeerCertFile
  224. if mem.Etcd.PeerCertFile != "" {
  225. var data []byte
  226. data, err = ioutil.ReadFile(mem.Etcd.PeerCertFile)
  227. if err != nil {
  228. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerCertFile, err)
  229. }
  230. clus.Members[i].PeerCertData = string(data)
  231. }
  232. clus.Members[i].PeerKeyPath = mem.Etcd.PeerKeyFile
  233. if mem.Etcd.PeerKeyFile != "" {
  234. var data []byte
  235. data, err = ioutil.ReadFile(mem.Etcd.PeerKeyFile)
  236. if err != nil {
  237. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerKeyFile, err)
  238. }
  239. clus.Members[i].PeerCertData = string(data)
  240. }
  241. clus.Members[i].PeerTrustedCAPath = mem.Etcd.PeerTrustedCAFile
  242. if mem.Etcd.PeerTrustedCAFile != "" {
  243. var data []byte
  244. data, err = ioutil.ReadFile(mem.Etcd.PeerTrustedCAFile)
  245. if err != nil {
  246. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerTrustedCAFile, err)
  247. }
  248. clus.Members[i].PeerCertData = string(data)
  249. }
  250. }
  251. clientTLS := mem.Etcd.ClientAutoTLS ||
  252. (mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" && mem.Etcd.ClientKeyFile != "" && mem.Etcd.ClientTrustedCAFile != "")
  253. if clientTLS {
  254. for _, cu := range mem.Etcd.ListenClientURLs {
  255. var u *url.URL
  256. u, err = url.Parse(cu)
  257. if err != nil {
  258. return nil, err
  259. }
  260. if u.Scheme != "https" { // TODO: support unix
  261. return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
  262. }
  263. }
  264. for _, cu := range mem.Etcd.AdvertiseClientURLs {
  265. var u *url.URL
  266. u, err = url.Parse(cu)
  267. if err != nil {
  268. return nil, err
  269. }
  270. if u.Scheme != "https" { // TODO: support unix
  271. return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
  272. }
  273. }
  274. clus.Members[i].ClientCertPath = mem.Etcd.ClientCertFile
  275. if mem.Etcd.ClientCertFile != "" {
  276. var data []byte
  277. data, err = ioutil.ReadFile(mem.Etcd.ClientCertFile)
  278. if err != nil {
  279. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientCertFile, err)
  280. }
  281. clus.Members[i].ClientCertData = string(data)
  282. }
  283. clus.Members[i].ClientKeyPath = mem.Etcd.ClientKeyFile
  284. if mem.Etcd.ClientKeyFile != "" {
  285. var data []byte
  286. data, err = ioutil.ReadFile(mem.Etcd.ClientKeyFile)
  287. if err != nil {
  288. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientKeyFile, err)
  289. }
  290. clus.Members[i].ClientCertData = string(data)
  291. }
  292. clus.Members[i].ClientTrustedCAPath = mem.Etcd.ClientTrustedCAFile
  293. if mem.Etcd.ClientTrustedCAFile != "" {
  294. var data []byte
  295. data, err = ioutil.ReadFile(mem.Etcd.ClientTrustedCAFile)
  296. if err != nil {
  297. return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientTrustedCAFile, err)
  298. }
  299. clus.Members[i].ClientCertData = string(data)
  300. }
  301. }
  302. }
  303. if len(clus.Tester.Cases) == 0 {
  304. return nil, errors.New("Cases not found")
  305. }
  306. if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 {
  307. return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
  308. }
  309. if clus.Tester.UpdatedDelayLatencyMs == 0 {
  310. clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs
  311. }
  312. for _, v := range clus.Tester.Cases {
  313. if _, ok := rpcpb.Case_value[v]; !ok {
  314. return nil, fmt.Errorf("%q is not defined in 'rpcpb.Case_value'", v)
  315. }
  316. }
  317. for _, v := range clus.Tester.Stressers {
  318. if _, ok := rpcpb.Stresser_value[v]; !ok {
  319. return nil, fmt.Errorf("Stresser is unknown; got %q", v)
  320. }
  321. }
  322. for _, v := range clus.Tester.Checkers {
  323. if _, ok := rpcpb.Checker_value[v]; !ok {
  324. return nil, fmt.Errorf("Checker is unknown; got %q", v)
  325. }
  326. }
  327. if clus.Tester.StressKeySuffixRangeTxn > 100 {
  328. return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn)
  329. }
  330. if clus.Tester.StressKeyTxnOps > 64 {
  331. return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps)
  332. }
  333. return clus, err
  334. }