|
| 1 | +/* |
| 2 | + * |
| 3 | + * Copyright 2023 gRPC authors. |
| 4 | + * |
| 5 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | + * you may not use this file except in compliance with the License. |
| 7 | + * You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + * |
| 17 | + */ |
| 18 | + |
| 19 | +package customroundrobin |
| 20 | + |
| 21 | +import ( |
| 22 | + "encoding/json" |
| 23 | + "fmt" |
| 24 | + "sync/atomic" |
| 25 | + |
| 26 | + "google.golang.org/grpc" |
| 27 | + "google.golang.org/grpc/balancer" |
| 28 | + "google.golang.org/grpc/connectivity" |
| 29 | + "google.golang.org/grpc/grpclog" |
| 30 | + "google.golang.org/grpc/resolver" |
| 31 | + "google.golang.org/grpc/serviceconfig" |
| 32 | +) |
| 33 | + |
| 34 | +func init() { |
| 35 | + balancer.Register(customRoundRobinBuilder{}) |
| 36 | +} |
| 37 | + |
| 38 | +const customRRName = "custom_round_robin" |
| 39 | + |
| 40 | +type customRRConfig struct { |
| 41 | + serviceconfig.LoadBalancingConfig `json:"-"` |
| 42 | + |
| 43 | + // N represents how often pick iterations chose the second SubConn in the |
| 44 | + // list. Defaults to 3. If 0 never choses second SubConn. |
| 45 | + N uint32 `json:"n,omitempty"` |
| 46 | +} |
| 47 | + |
| 48 | +type customRoundRobinBuilder struct{} |
| 49 | + |
| 50 | +func (customRoundRobinBuilder) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { |
| 51 | + lbConfig := &customRRConfig{ |
| 52 | + N: 3, |
| 53 | + } |
| 54 | + if err := json.Unmarshal(s, lbConfig); err != nil { |
| 55 | + return nil, fmt.Errorf("custom-round-robin: unable to unmarshal customRRConfig: %v", err) |
| 56 | + } |
| 57 | + return lbConfig, nil |
| 58 | +} |
| 59 | + |
| 60 | +func (customRoundRobinBuilder) Name() string { |
| 61 | + return customRRName |
| 62 | +} |
| 63 | + |
| 64 | +func (customRoundRobinBuilder) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { |
| 65 | + pfBuilder := balancer.Get(grpc.PickFirstBalancerName) |
| 66 | + if pfBuilder == nil { |
| 67 | + return nil |
| 68 | + } |
| 69 | + return &customRoundRobin{ |
| 70 | + cc: cc, |
| 71 | + bOpts: bOpts, |
| 72 | + pfs: resolver.NewEndpointMap(), |
| 73 | + pickFirstBuilder: pfBuilder, |
| 74 | + } |
| 75 | +} |
| 76 | + |
| 77 | +var logger = grpclog.Component("example") |
| 78 | + |
| 79 | +type customRoundRobin struct { |
| 80 | + // All state and operations on this balancer are either initialized at build |
| 81 | + // time and read only after, or are only accessed as part of it's |
| 82 | + // balancer.Balancer API (UpdateState from children only comes in from |
| 83 | + // balancer.Balancer calls as well, and children are called one at a time), |
| 84 | + // in which calls are guaranteed to come synchronously. Thus, no extra |
| 85 | + // synchronization is required in this balancer. |
| 86 | + cc balancer.ClientConn |
| 87 | + bOpts balancer.BuildOptions |
| 88 | + // Note that this balancer is a petiole policy which wraps pick first (see |
| 89 | + // gRFC A61). This is the intended way a user written custom lb should be |
| 90 | + // specified, as pick first will contain a lot of useful functionality, such |
| 91 | + // as Sticky Transient Failure, Happy Eyeballs, and Health Checking. |
| 92 | + pickFirstBuilder balancer.Builder |
| 93 | + pfs *resolver.EndpointMap |
| 94 | + |
| 95 | + n uint32 |
| 96 | + inhibitPickerUpdates bool |
| 97 | +} |
| 98 | + |
| 99 | +func (crr *customRoundRobin) UpdateClientConnState(state balancer.ClientConnState) error { |
| 100 | + if logger.V(2) { |
| 101 | + logger.Info("custom_round_robin: got new ClientConn state: ", state) |
| 102 | + } |
| 103 | + crrCfg, ok := state.BalancerConfig.(*customRRConfig) |
| 104 | + if !ok { |
| 105 | + return balancer.ErrBadResolverState |
| 106 | + } |
| 107 | + crr.n = crrCfg.N |
| 108 | + |
| 109 | + endpointSet := resolver.NewEndpointMap() |
| 110 | + crr.inhibitPickerUpdates = true |
| 111 | + for _, endpoint := range state.ResolverState.Endpoints { |
| 112 | + endpointSet.Set(endpoint, nil) |
| 113 | + var pickFirst *balancerWrapper |
| 114 | + pf, ok := crr.pfs.Get(endpoint) |
| 115 | + if ok { |
| 116 | + pickFirst = pf.(*balancerWrapper) |
| 117 | + } else { |
| 118 | + pickFirst = &balancerWrapper{ |
| 119 | + ClientConn: crr.cc, |
| 120 | + crr: crr, |
| 121 | + } |
| 122 | + pfb := crr.pickFirstBuilder.Build(pickFirst, crr.bOpts) |
| 123 | + pickFirst.Balancer = pfb |
| 124 | + crr.pfs.Set(endpoint, pickFirst) |
| 125 | + } |
| 126 | + // Update child uncondtionally, in case attributes or address ordering |
| 127 | + // changed. Let pick first deal with any potential diffs, too |
| 128 | + // complicated to only update if we know something changed. |
| 129 | + pickFirst.UpdateClientConnState(balancer.ClientConnState{ |
| 130 | + ResolverState: resolver.State{ |
| 131 | + Endpoints: []resolver.Endpoint{endpoint}, |
| 132 | + Attributes: state.ResolverState.Attributes, |
| 133 | + }, |
| 134 | + // no service config, never needed to turn on address list shuffling |
| 135 | + // bool in petiole policies. |
| 136 | + }) |
| 137 | + // Ignore error because just care about ready children. |
| 138 | + } |
| 139 | + for _, e := range crr.pfs.Keys() { |
| 140 | + ep, _ := crr.pfs.Get(e) |
| 141 | + pickFirst := ep.(balancer.Balancer) |
| 142 | + // pick first was removed by resolver (unique endpoint logically |
| 143 | + // corresponding to pick first child was removed). |
| 144 | + if _, ok := endpointSet.Get(e); !ok { |
| 145 | + pickFirst.Close() |
| 146 | + crr.pfs.Delete(e) |
| 147 | + } |
| 148 | + } |
| 149 | + crr.inhibitPickerUpdates = false |
| 150 | + crr.regeneratePicker() // one synchronous picker update per Update Client Conn State operation. |
| 151 | + return nil |
| 152 | +} |
| 153 | + |
| 154 | +func (crr *customRoundRobin) ResolverError(err error) { |
| 155 | + crr.inhibitPickerUpdates = true |
| 156 | + for _, pf := range crr.pfs.Values() { |
| 157 | + pickFirst := pf.(*balancerWrapper) |
| 158 | + pickFirst.ResolverError(err) |
| 159 | + } |
| 160 | + crr.inhibitPickerUpdates = false |
| 161 | + crr.regeneratePicker() |
| 162 | +} |
| 163 | + |
| 164 | +// This function is deprecated. SubConn state updates now come through listener |
| 165 | +// callbacks. This balancer does not deal with SubConns directly and has no need |
| 166 | +// to intercept listener callbacks. |
| 167 | +func (crr *customRoundRobin) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { |
| 168 | + logger.Errorf("custom_round_robin: UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) |
| 169 | +} |
| 170 | + |
| 171 | +func (crr *customRoundRobin) Close() { |
| 172 | + for _, pf := range crr.pfs.Values() { |
| 173 | + pickFirst := pf.(balancer.Balancer) |
| 174 | + pickFirst.Close() |
| 175 | + } |
| 176 | +} |
| 177 | + |
| 178 | +// regeneratePicker generates a picker based off persisted child balancer state |
| 179 | +// and forwards it upward. This is intended to be fully executed once per |
| 180 | +// relevant balancer.Balancer operation into custom round robin balancer. |
| 181 | +func (crr *customRoundRobin) regeneratePicker() { |
| 182 | + if crr.inhibitPickerUpdates { |
| 183 | + return |
| 184 | + } |
| 185 | + |
| 186 | + var readyPickers []balancer.Picker |
| 187 | + for _, bw := range crr.pfs.Values() { |
| 188 | + pickFirst := bw.(*balancerWrapper) |
| 189 | + if pickFirst.state.ConnectivityState == connectivity.Ready { |
| 190 | + readyPickers = append(readyPickers, pickFirst.state.Picker) |
| 191 | + } |
| 192 | + } |
| 193 | + |
| 194 | + // For determinism, this balancer only updates it's picker when both |
| 195 | + // backends of the example are ready. Thus, no need to keep track of |
| 196 | + // aggregated state and can simply specify this balancer is READY once it |
| 197 | + // has two ready children. |
| 198 | + if len(readyPickers) != 2 { |
| 199 | + return |
| 200 | + } |
| 201 | + picker := &customRoundRobinPicker{ |
| 202 | + pickers: readyPickers, |
| 203 | + n: crr.n, |
| 204 | + next: 0, |
| 205 | + } |
| 206 | + crr.cc.UpdateState(balancer.State{ |
| 207 | + ConnectivityState: connectivity.Ready, |
| 208 | + Picker: picker, |
| 209 | + }) |
| 210 | +} |
| 211 | + |
| 212 | +type balancerWrapper struct { |
| 213 | + balancer.Balancer // Simply forward balancer.Balancer operations |
| 214 | + balancer.ClientConn // embed to intercept UpdateState, doesn't deal with SubConns |
| 215 | + |
| 216 | + crr *customRoundRobin |
| 217 | + |
| 218 | + state balancer.State |
| 219 | +} |
| 220 | + |
| 221 | +// Picker updates from pick first are all triggered by synchronous calls down |
| 222 | +// into balancer.Balancer (client conn state updates, resolver errors, subconn |
| 223 | +// state updates (through listener callbacks, which is still treated as part of |
| 224 | +// balancer API)). |
| 225 | +func (bw *balancerWrapper) UpdateState(state balancer.State) { |
| 226 | + bw.state = state |
| 227 | + // Calls back into this inline will be inhibited when part of |
| 228 | + // UpdateClientConnState() and ResolverError(), and regenerate picker will |
| 229 | + // be called manually at the end of those operations. However, for |
| 230 | + // UpdateSubConnState() and subsequent UpdateState(), this needs to update |
| 231 | + // picker, so call this regeneratePicker() here. |
| 232 | + bw.crr.regeneratePicker() |
| 233 | +} |
| 234 | + |
| 235 | +type customRoundRobinPicker struct { |
| 236 | + pickers []balancer.Picker |
| 237 | + n uint32 |
| 238 | + next uint32 |
| 239 | +} |
| 240 | + |
| 241 | +func (crrp *customRoundRobinPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { |
| 242 | + next := atomic.AddUint32(&crrp.next, 1) |
| 243 | + index := 0 |
| 244 | + if next != 0 && next%crrp.n == 0 { |
| 245 | + index = 1 |
| 246 | + } |
| 247 | + childPicker := crrp.pickers[index%len(crrp.pickers)] |
| 248 | + return childPicker.Pick(info) |
| 249 | +} |
0 commit comments