diff --git a/components/fl-operator/controllers/floperator_controller.go b/components/fl-operator/controllers/floperator_controller.go index f657e06..d5f31fe 100644 --- a/components/fl-operator/controllers/floperator_controller.go +++ b/components/fl-operator/controllers/floperator_controller.go @@ -43,7 +43,6 @@ type FlOperatorReconciler struct { client.Client Scheme *runtime.Scheme EnvoyproxyConfigFile string - OrchestratorClient *orchestratorClient.Client } const ( @@ -107,26 +106,28 @@ func (r *FlOperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Create gRPC client for connection to FL-Orchestrator - if r.OrchestratorClient == nil { - log.Info("Creating OrchestratorClient") + log.Info("Creating OrchestratorClient") - timeout := time.Second * 30 // TODO move this into the spec? - serverAddress := fmt.Sprintf("%s.%s:9080", serviceNameEnvoyProxy, req.Namespace) - dialOptions := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - } - conn, err := grpc.Dial(serverAddress, dialOptions...) - if err != nil { - log.Error(err, "Could not create gRPC connection") - return ctrl.Result{}, err - } - - cl := orchestratorClient.NewClient(conn, timeout) - r.OrchestratorClient = &cl + timeout := time.Second * 30 // TODO move this into the spec? + serverAddress := fmt.Sprintf("%s.%s:9080", serviceNameEnvoyProxy, req.Namespace) + dialOptions := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), } + conn, err := grpc.Dial(serverAddress, dialOptions...) + if err != nil { + log.Error(err, "Could not create gRPC connection") + return ctrl.Result{}, err + } + + // We can run into situations where we have a gRPC connection, but aren't + // yet able to use it because the envoy proxy isn't fully set up yet. + // As a result, we create a new connection on every call to 'Reconcile'. + // Some connections might fail but these failures won't affect later calls + // to 'Reconcile' as we're not caching failed connections. + cl := orchestratorClient.NewClient(conn, timeout) // Fetch tasks from FL-Orchestrator - response, err := r.OrchestratorClient.GetTasks(ctx) + response, err := cl.GetTasks(ctx) if err != nil { log.Error(err, "Could not fetch tasks") return ctrl.Result{ diff --git a/components/fl-operator/main.go b/components/fl-operator/main.go index c1cf476..145d332 100644 --- a/components/fl-operator/main.go +++ b/components/fl-operator/main.go @@ -82,7 +82,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), EnvoyproxyConfigFile: "./deployment/envoyproxy.yaml.tpl", - OrchestratorClient: nil, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "FlOperator") os.Exit(1)