diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/util/signal.go b/cmd/util/signal.go new file mode 100644 index 0000000..fbe5d28 --- /dev/null +++ b/cmd/util/signal.go @@ -0,0 +1,62 @@ +package util + +import ( + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var errorExitCodeChannel = make(chan int, 1) + +// ExitBehavior controls how the program should be terminated +// in response to a shutdown signal. +type ExitBehavior int + +const ( + // AlwaysErrCode indicates the exit code of the program should always be nonzero + // and should correspond to the numeric value of the signal that was received. + AlwaysErrCode ExitBehavior = iota + + // GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating + // goroutines and returning an exit code of 0 if there are no errors during shutdown. + GracefulShutdown ExitBehavior = iota +) + +// SetupExitHandler: +// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM +// or SIGINT). If a second signal is caught, the program is terminated directly with +// exit code 130. +// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...) +// if there is a exit code in the errorExitCodeChannel. +// The errorExitCodeChannel receives exit codes when SetExitCode is called or when +// a shutdown signal is received (only if exitBehavior is AlwaysErrCode). +func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + // first signal. Close stop chan and pass exit code to exitCodeChannel. + exitCode := 128 + int((<-c).(syscall.Signal)) + if exitBehavior == AlwaysErrCode { + errorExitCodeChannel <- exitCode + } + close(stop) + // second signal. Exit directly. + <-c + os.Exit(130) + }() + + return stop, func() { + select { + case signal := <-errorExitCodeChannel: + os.Exit(signal) + default: + // Do not exit, there are no exit codes in the channel, + // so just continue and let the main function go out of + // scope instead. + } + } +} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/util/signal.go b/cmd/util/signal.go new file mode 100644 index 0000000..fbe5d28 --- /dev/null +++ b/cmd/util/signal.go @@ -0,0 +1,62 @@ +package util + +import ( + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var errorExitCodeChannel = make(chan int, 1) + +// ExitBehavior controls how the program should be terminated +// in response to a shutdown signal. +type ExitBehavior int + +const ( + // AlwaysErrCode indicates the exit code of the program should always be nonzero + // and should correspond to the numeric value of the signal that was received. + AlwaysErrCode ExitBehavior = iota + + // GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating + // goroutines and returning an exit code of 0 if there are no errors during shutdown. + GracefulShutdown ExitBehavior = iota +) + +// SetupExitHandler: +// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM +// or SIGINT). If a second signal is caught, the program is terminated directly with +// exit code 130. +// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...) +// if there is a exit code in the errorExitCodeChannel. +// The errorExitCodeChannel receives exit codes when SetExitCode is called or when +// a shutdown signal is received (only if exitBehavior is AlwaysErrCode). +func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + // first signal. Close stop chan and pass exit code to exitCodeChannel. + exitCode := 128 + int((<-c).(syscall.Signal)) + if exitBehavior == AlwaysErrCode { + errorExitCodeChannel <- exitCode + } + close(stop) + // second signal. Exit directly. + <-c + os.Exit(130) + }() + + return stop, func() { + select { + case signal := <-errorExitCodeChannel: + os.Exit(signal) + default: + // Do not exit, there are no exit codes in the channel, + // so just continue and let the main function go out of + // scope instead. + } + } +} diff --git a/cmd/util/signal_posix.go b/cmd/util/signal_posix.go new file mode 100644 index 0000000..dd89878 --- /dev/null +++ b/cmd/util/signal_posix.go @@ -0,0 +1,8 @@ +package util + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/util/signal.go b/cmd/util/signal.go new file mode 100644 index 0000000..fbe5d28 --- /dev/null +++ b/cmd/util/signal.go @@ -0,0 +1,62 @@ +package util + +import ( + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var errorExitCodeChannel = make(chan int, 1) + +// ExitBehavior controls how the program should be terminated +// in response to a shutdown signal. +type ExitBehavior int + +const ( + // AlwaysErrCode indicates the exit code of the program should always be nonzero + // and should correspond to the numeric value of the signal that was received. + AlwaysErrCode ExitBehavior = iota + + // GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating + // goroutines and returning an exit code of 0 if there are no errors during shutdown. + GracefulShutdown ExitBehavior = iota +) + +// SetupExitHandler: +// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM +// or SIGINT). If a second signal is caught, the program is terminated directly with +// exit code 130. +// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...) +// if there is a exit code in the errorExitCodeChannel. +// The errorExitCodeChannel receives exit codes when SetExitCode is called or when +// a shutdown signal is received (only if exitBehavior is AlwaysErrCode). +func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + // first signal. Close stop chan and pass exit code to exitCodeChannel. + exitCode := 128 + int((<-c).(syscall.Signal)) + if exitBehavior == AlwaysErrCode { + errorExitCodeChannel <- exitCode + } + close(stop) + // second signal. Exit directly. + <-c + os.Exit(130) + }() + + return stop, func() { + select { + case signal := <-errorExitCodeChannel: + os.Exit(signal) + default: + // Do not exit, there are no exit codes in the channel, + // so just continue and let the main function go out of + // scope instead. + } + } +} diff --git a/cmd/util/signal_posix.go b/cmd/util/signal_posix.go new file mode 100644 index 0000000..dd89878 --- /dev/null +++ b/cmd/util/signal_posix.go @@ -0,0 +1,8 @@ +package util + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/go.mod b/go.mod index 7895144..dd13edf 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/pavlo-v-chernykh/keystore-go/v4 v4.4.0 + github.com/spf13/cobra v1.4.0 + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 k8s.io/client-go v0.25.0 @@ -46,6 +48,7 @@ github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/util/signal.go b/cmd/util/signal.go new file mode 100644 index 0000000..fbe5d28 --- /dev/null +++ b/cmd/util/signal.go @@ -0,0 +1,62 @@ +package util + +import ( + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var errorExitCodeChannel = make(chan int, 1) + +// ExitBehavior controls how the program should be terminated +// in response to a shutdown signal. +type ExitBehavior int + +const ( + // AlwaysErrCode indicates the exit code of the program should always be nonzero + // and should correspond to the numeric value of the signal that was received. + AlwaysErrCode ExitBehavior = iota + + // GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating + // goroutines and returning an exit code of 0 if there are no errors during shutdown. + GracefulShutdown ExitBehavior = iota +) + +// SetupExitHandler: +// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM +// or SIGINT). If a second signal is caught, the program is terminated directly with +// exit code 130. +// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...) +// if there is a exit code in the errorExitCodeChannel. +// The errorExitCodeChannel receives exit codes when SetExitCode is called or when +// a shutdown signal is received (only if exitBehavior is AlwaysErrCode). +func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + // first signal. Close stop chan and pass exit code to exitCodeChannel. + exitCode := 128 + int((<-c).(syscall.Signal)) + if exitBehavior == AlwaysErrCode { + errorExitCodeChannel <- exitCode + } + close(stop) + // second signal. Exit directly. + <-c + os.Exit(130) + }() + + return stop, func() { + select { + case signal := <-errorExitCodeChannel: + os.Exit(signal) + default: + // Do not exit, there are no exit codes in the channel, + // so just continue and let the main function go out of + // scope instead. + } + } +} diff --git a/cmd/util/signal_posix.go b/cmd/util/signal_posix.go new file mode 100644 index 0000000..dd89878 --- /dev/null +++ b/cmd/util/signal_posix.go @@ -0,0 +1,8 @@ +package util + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/go.mod b/go.mod index 7895144..dd13edf 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/pavlo-v-chernykh/keystore-go/v4 v4.4.0 + github.com/spf13/cobra v1.4.0 + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 k8s.io/client-go v0.25.0 @@ -46,6 +48,7 @@ github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/go.sum b/go.sum index df924a3..f424c66 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -232,6 +233,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -314,11 +317,14 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -469,6 +475,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go new file mode 100644 index 0000000..4a80903 --- /dev/null +++ b/cmd/controller/app/controller.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "fmt" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "golang.org/x/sync/errgroup" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/clock" +) + +func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error { + rootCtx, cancelContext := context.WithCancel(util.ContextWithStopCh(context.Background(), stopCh)) + defer cancelContext() + + rootCtx = logf.NewContext(rootCtx, logf.Log, "controller") + log := logf.FromContext(rootCtx) + g, rootCtx := errgroup.WithContext(rootCtx) + + ctxFactory, err := buildControllerContextFactory(rootCtx, opts) + if err != nil { + return err + } + + // Build the base controller context for the cert-manager controller manager + // used here. + ctx, err := ctxFactory.Build() + if err != nil { + return err + } + + enabledControllers := opts.EnabledControllers() + logf.Log.V(logf.DebugLevel).Info(fmt.Sprintf("enabled controllers: %s", enabledControllers)) + + //TODO: Start metrics server + //TODO: Start profiler + //TODO: Start leader elect + + for n, fn := range controller.Known() { + log := log.WithValues("controller", n) + + // only run a controller if it is been enabled + if !enabledControllers.Has(n) { + log.V(logf.InfoLevel).Info("not starting controller as it's disabled") + continue + } + + iface, err := fn(ctxFactory) + if err != nil { + if err != nil { + err = fmt.Errorf("error starting controller: %v", err) + cancelContext() + if err1 := g.Wait(); err1 != nil { + return utilerrors.NewAggregate([]error{err, err1}) + } + return err + } + } + + g.Go(func() error { + log.V(logf.InfoLevel).Info("starting controller") + + // TODO: make this either a constant or a command line flag + workers := 5 + return iface.Run(workers, rootCtx.Done()) + }) + } + + log.V(logf.DebugLevel).Info("starting shared informer factories") + ctx.SharedInformerFactory.Start(rootCtx.Done()) + ctx.KubeSharedInformerFactory.Start(rootCtx.Done()) + + err = g.Wait() + if err != nil { + return fmt.Errorf("error starting controller: %v", err) + } + log.V(logf.InfoLevel).Info("control loops exited") + + return nil +} + +func buildControllerContextFactory(ctx context.Context, opts *options.ControllerOptions) (*controller.ContextFactory, error) { + // log := logf.FromContext(ctx) + + ctxFactory, err := controller.NewContextFactory(ctx, controller.ContextOptions{ + Clock: clock.RealClock{}, + }) + + if err != nil { + return nil, err + } + + return ctxFactory, nil +} diff --git a/cmd/controller/app/options/options.go b/cmd/controller/app/options/options.go new file mode 100644 index 0000000..1f05612 --- /dev/null +++ b/cmd/controller/app/options/options.go @@ -0,0 +1,81 @@ +package options + +import ( + "fmt" + "strings" + + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates/issuing" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" +) + +type ControllerOptions struct { + MetricsBindAddress string + HealthProbeBindAddress string + LeaderElection bool + Controllers []string +} + +var ( + allControllers = []string{ + // certificate controllers + issuing.ControllerName, + } + + defaultEnabledControllers = []string{ + issuing.ControllerName, + } +) + +func NewControllerOptions() *ControllerOptions { + return &ControllerOptions{ + Controllers: defaultEnabledControllers, + } + +} + +func getStrSlice(s string) []string { + var arr = strings.Split(s, ",") + ret := make([]string, len(arr)) + for _, str := range arr { + ret = append(ret, strings.TrimSpace(str)) + } + return ret +} + +func (s *ControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.HealthProbeBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + fs.StringVar(&s.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&s.LeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + var controllerSet string + fs.StringVar(&controllerSet, "controllers", "", fmt.Sprintf(""+ + "A list of controllers to enable. '--controllers=*' enables all "+ + "on-by-default controllers, '--controllers=foo' enables just the controller "+ + "named 'foo', '--controllers=*,-foo' disables the controller named "+ + "'foo'.\nAll controllers: %s", + strings.Join(allControllers, ", "))) + s.Controllers = getStrSlice(controllerSet) +} + +func (o *ControllerOptions) EnabledControllers() sets.String { + enabled := sets.NewString() + + for _, controller := range o.Controllers { + switch { + // Enable all controllers + case controller == "*": + enabled = enabled.Insert(defaultEnabledControllers...) + default: + enabled = enabled.Insert(controller) + + } + } + return enabled +} + +// TODO: Implement validation +func (o *ControllerOptions) Validate() error { + return nil +} diff --git a/cmd/controller/app/start.go b/cmd/controller/app/start.go new file mode 100644 index 0000000..fcab846 --- /dev/null +++ b/cmd/controller/app/start.go @@ -0,0 +1,66 @@ +package app + +import ( + "fmt" + + options "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app/options" + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +type AnthosCertManagerControllerOptions struct { + ControllerOptions *options.ControllerOptions +} + +func NewAnthosCertManagerControllerOptions() *AnthosCertManagerControllerOptions { + o := &AnthosCertManagerControllerOptions{ + ControllerOptions: options.NewControllerOptions(), + } + return o +} + +// NewCommandStartCertManagerController is a CLI handler for starting cert-manager +func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command { + o := NewAnthosCertManagerControllerOptions() + cmd := &cobra.Command{ + Use: "cert-manager-controller", + Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit), + Long: ` +cert-manager is a Kubernetes addon to automate the management and issuance of +TLS certificates from various issuing sources. +It will ensure certificates are valid and up to date periodically, and attempt +to renew certificates at an appropriate time before expiry.`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(args); err != nil { + return fmt.Errorf("error validating options: %s", err) + } + + logf.Log.V(logf.InfoLevel).Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit) + if err := o.RunCertManagerController(stopCh); err != nil { + cmd.SilenceUsage = true // Don't display usage information when exiting because of an error + return err + } + + return nil + }, + SilenceErrors: true, // Errors are already logged when calling cmd.Execute() + } + + flags := cmd.Flags() + o.ControllerOptions.AddFlags(flags) + + return cmd + +} +func (o AnthosCertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) error { + return Run(o.ControllerOptions, stopCh) +} + +func (o AnthosCertManagerControllerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.ControllerOptions.Validate()) + return utilerrors.NewAggregate(errors) +} diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 0000000..28efd6b --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,29 @@ +package controller + +import ( + "flag" + + app "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller/app" + "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/util" + + logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" +) + +// Init function to start the controller +func Init() { + stopCh, exit := util.SetupExitHandler(util.GracefulShutdown) + defer exit() + + logf.InitLogs(flag.CommandLine) + defer logf.FlushLogs() + + cmd := app.NewCommandStartCertManagerController(stopCh) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + + flag.CommandLine.Parse([]string{}) + + if err := cmd.Execute(); err != nil { + logf.Log.Error(err, "error while executing") + util.SetExitCode(err) + } +} diff --git a/cmd/util/context.go b/cmd/util/context.go new file mode 100644 index 0000000..e33eca3 --- /dev/null +++ b/cmd/util/context.go @@ -0,0 +1,18 @@ +package util + +import "context" + +// ContextWithStopCh will wrap a context with a stop channel. +// When the provided stopCh closes, the cancel() will be called on the context. +// This provides a convenient way to represent a stop channel as a context. +func ContextWithStopCh(ctx context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(ctx) + go func() { + defer cancel() + select { + case <-ctx.Done(): + case <-stopCh: + } + }() + return ctx +} diff --git a/cmd/util/exit.go b/cmd/util/exit.go new file mode 100644 index 0000000..73d70a9 --- /dev/null +++ b/cmd/util/exit.go @@ -0,0 +1,13 @@ +package util + +import ( + "context" + "errors" +) + +// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error. +func SetExitCode(err error) { + if (err != nil) && !errors.Is(err, context.Canceled) { + errorExitCodeChannel <- 1 // Indicate that there was an error + } +} diff --git a/cmd/util/signal.go b/cmd/util/signal.go new file mode 100644 index 0000000..fbe5d28 --- /dev/null +++ b/cmd/util/signal.go @@ -0,0 +1,62 @@ +package util + +import ( + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var errorExitCodeChannel = make(chan int, 1) + +// ExitBehavior controls how the program should be terminated +// in response to a shutdown signal. +type ExitBehavior int + +const ( + // AlwaysErrCode indicates the exit code of the program should always be nonzero + // and should correspond to the numeric value of the signal that was received. + AlwaysErrCode ExitBehavior = iota + + // GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating + // goroutines and returning an exit code of 0 if there are no errors during shutdown. + GracefulShutdown ExitBehavior = iota +) + +// SetupExitHandler: +// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM +// or SIGINT). If a second signal is caught, the program is terminated directly with +// exit code 130. +// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...) +// if there is a exit code in the errorExitCodeChannel. +// The errorExitCodeChannel receives exit codes when SetExitCode is called or when +// a shutdown signal is received (only if exitBehavior is AlwaysErrCode). +func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + // first signal. Close stop chan and pass exit code to exitCodeChannel. + exitCode := 128 + int((<-c).(syscall.Signal)) + if exitBehavior == AlwaysErrCode { + errorExitCodeChannel <- exitCode + } + close(stop) + // second signal. Exit directly. + <-c + os.Exit(130) + }() + + return stop, func() { + select { + case signal := <-errorExitCodeChannel: + os.Exit(signal) + default: + // Do not exit, there are no exit codes in the channel, + // so just continue and let the main function go out of + // scope instead. + } + } +} diff --git a/cmd/util/signal_posix.go b/cmd/util/signal_posix.go new file mode 100644 index 0000000..dd89878 --- /dev/null +++ b/cmd/util/signal_posix.go @@ -0,0 +1,8 @@ +package util + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/go.mod b/go.mod index 7895144..dd13edf 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/pavlo-v-chernykh/keystore-go/v4 v4.4.0 + github.com/spf13/cobra v1.4.0 + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 k8s.io/client-go v0.25.0 @@ -46,6 +48,7 @@ github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/go.sum b/go.sum index df924a3..f424c66 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -232,6 +233,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -314,11 +317,14 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -469,6 +475,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/main.go b/main.go index b995425..c74e09b 100644 --- a/main.go +++ b/main.go @@ -16,102 +16,8 @@ package main -import ( - "flag" - "os" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - anthoscertmanager "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/apis/anthoscertmanager/v1" - logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(anthoscertmanager.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} +import "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/cmd/controller" func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - logf.InitLogs(flag.CommandLine) - defer logf.FlushLogs() - - ctrl.SetLogger(logf.Log) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "bef4c06f.onprem.cluster.gke.io", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - // if err = (&certificates.CertificateReconciler{ - // Client: mgr.GetClient(), - // Scheme: mgr.GetScheme(), - // }).SetupWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create controller", "controller", "Certificate") - // os.Exit(1) - // } - //+kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } + controller.Init() }