diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..15fa3b3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# Build the manager binary +FROM docker.m.daocloud.io/golang:1.18.1 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY commands/ commands/ +COPY controllers/ controllers/ +COPY errors/ errors/ +COPY operations/ operations/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER nonroot:nonroot + +ENTRYPOINT ["/manager"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..32b7182 --- /dev/null +++ b/Makefile @@ -0,0 +1,73 @@ + +# Image URL to use all building/pushing image targets +IMG ?= daocloud.io/daocloud/kubeadm-operator:v0.0.2 +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd" + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +all: manager + +# Run tests +test: generate fmt vet manifests + go test ./... -coverprofile cover.out + +# Build manager binary +manager: generate fmt vet + go build -o bin/manager main.go + +# Run against the configured Kubernetes cluster in ~/.kube/config +run: generate fmt vet manifests + go run ./main.go + +# Install CRDs into a cluster +install: manifests + kustomize build config/crd | kubectl apply -f - + +# Deploy controller in the configured Kubernetes cluster in ~/.kube/config +deploy: manifests + cd config/manager && kustomize edit set image controller=${IMG} + kustomize build config/default | kubectl apply -f - + +undeploy: manifests + cd config/manager && kustomize edit set image controller=${IMG} + kustomize build config/default | kubectl delete -f - + +# Generate manifests e.g. CRD, RBAC etc. +manifests: controller-gen + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +# Run go fmt against code +fmt: + go fmt ./... + +# Run go vet against code +vet: + go vet ./... + +# Generate code +generate: controller-gen + $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..." + +# Build the docker image +docker-build: test + docker build . -t ${IMG} + +# Push the docker image +docker-push: + docker push ${IMG} + +# find or download controller-gen +# download controller-gen if necessary +controller-gen: +ifeq (, $(shell which controller-gen)) + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0 +CONTROLLER_GEN=$(GOBIN)/controller-gen +else +CONTROLLER_GEN=$(shell which controller-gen) +endif diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..b1a8720 --- /dev/null +++ b/PROJECT @@ -0,0 +1,13 @@ +version: "2" +domain: kubeadm.x-k8s.io +repo: k8s.io/kubeadm/operator +resources: +- group: operator + version: v1alpha1 + kind: Operation +- group: operator + version: v1alpha1 + kind: RuntimeTaskGroup +- group: operator + version: v1alpha1 + kind: RuntimeTask diff --git a/README.md b/README.md new file mode 100644 index 0000000..d6df5d8 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# Kubeadm operator + +The kubeadm-operator is an experimental project, still WIP. +Do not use in production. + +See [KEP](https://git.k8s.io/enhancements/keps/sig-cluster-lifecycle/kubeadm/2505-Kubeadm-operator) for more details. + +## Quick Start + +Configure kubeconfig for your cluster. + +``` +git clone git@github.com:pacoxu/kubeadm-operator.git +cd kubeadm-operator +make install +make deploy +``` + +## Demo + +After installation, a deploy named `operator-controller-manager` is running in namespace `operator-system`. +``` +[root@daocloud ~]# kubectl get pod -n operator-system -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +operator-controller-manager-64c448f5b-p682x 2/2 Running 0 77m 172.32.230.197 daocloud +``` + +If you create a dry-run upgrade operation, there will be a runtimetaqskgroup with + +``` +[root@daocloud ~]# cat up.yaml +apiVersion: operator.kubeadm.x-k8s.io/v1alpha1 +kind: Operation +metadata: + name: upgrade-1 +spec: + executionMode: DryRun + upgrade: + kubernetesVersion: v1.23.4 + +[root@daocloud ~]# kubectl get runtimetaskgroup -w +NAME PHASE NODES SUCCEEDED FAILED +upgrade-1-01-upgrade-cp-1 Running 1 +upgrade-1-01-upgrade-cp-1 Succeeded 1 1 +upgrade-1-02-upgrade-cp-n +upgrade-1-02-upgrade-cp-n Running +upgrade-1-02-upgrade-cp-n Succeeded +upgrade-1-02-upgrade-w +upgrade-1-02-upgrade-w Running +upgrade-1-02-upgrade-w Succeeded +upgrade-1-02-upgrade-w Succeeded +``` + +After the operation is done, the operation and task group are all `Succeeded`. + +``` +[root@daocloud ~]# kubectl get operations +NAME PHASE GROUPS SUCCEEDED FAILED +upgrade-1 Succeeded 3 3 +[root@daocloud ~]# kubectl get runtimetaskgroup +NAME PHASE NODES SUCCEEDED FAILED +upgrade-1-01-upgrade-cp-1 Succeeded 1 1 +upgrade-1-02-upgrade-cp-n Succeeded +upgrade-1-02-upgrade-w Succeeded +[root@daocloud ~]# kubectl get runtimetask +NAME PHASE STARTTIME COMMAND COMPLETIONTIME +upgrade-1-01-upgrade-cp-1-daocloud Succeeded 75m 3/3 75m + + +``` diff --git a/api/v1alpha1/command_descriptor_types.go b/api/v1alpha1/command_descriptor_types.go new file mode 100644 index 0000000..cba0ed3 --- /dev/null +++ b/api/v1alpha1/command_descriptor_types.go @@ -0,0 +1,129 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// CommandDescriptor represents a command to be performed. +// Only one of its members may be specified. +type CommandDescriptor struct { + + // +optional + KubeadmRenewCertificates *KubeadmRenewCertsCommandSpec `json:"kubeadmRenewCertificates,omitempty"` + + // +optional + KubeadmUpgradeApply *KubeadmUpgradeApplyCommandSpec `json:"kubeadmUpgradeApply,omitempty"` + + // +optional + KubeadmUpgradeNode *KubeadmUpgradeNodeCommandSpec `json:"kubeadmUpgradeNode,omitempty"` + + // +optional + Preflight *PreflightCommandSpec `json:"preflight,omitempty"` + + // +optional + UpgradeKubeadm *UpgradeKubeadmCommandSpec `json:"upgradeKubeadm,omitempty"` + + // +optional + UpgradeKubeletAndKubeactl *UpgradeKubeletAndKubeactlCommandSpec `json:"upgradeKubeletAndKubeactl,omitempty"` + + // +optional + KubectlDrain *KubectlDrainCommandSpec `json:"kubectlDrain,omitempty"` + + // +optional + KubectlUncordon *KubectlUncordonCommandSpec `json:"kubectlUncordon,omitempty"` + + // Pass provide a dummy command for testing the kubeadm-operator workflow. + // +optional + Pass *PassCommandSpec `json:"pass,omitempty"` + + // Fail provide a dummy command for testing the kubeadm-operator workflow. + // +optional + Fail *FailCommandSpec `json:"fail,omitempty"` + + // Wait pauses the execution on the next command for a given number of seconds. + // +optional + Wait *WaitCommandSpec `json:"wait,omitempty"` +} + +// PreflightCommandSpec provides... +type PreflightCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// UpgradeKubeadmCommandSpec provides... +type UpgradeKubeadmCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// KubeadmUpgradeApplyCommandSpec provides... +type KubeadmUpgradeApplyCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// KubeadmUpgradeNodeCommandSpec provides... +type KubeadmUpgradeNodeCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// KubectlDrainCommandSpec provides... +type KubectlDrainCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// KubectlUncordonCommandSpec provides... +type KubectlUncordonCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// UpgradeKubeletAndKubeactlCommandSpec provides... +type UpgradeKubeletAndKubeactlCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// KubeadmRenewCertsCommandSpec provides... +type KubeadmRenewCertsCommandSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// PassCommandSpec provide a dummy command for testing the kubeadm-operator workflow. +type PassCommandSpec struct { +} + +// FailCommandSpec provide a dummy command for testing the kubeadm-operator workflow. +type FailCommandSpec struct { +} + +// WaitCommandSpec pauses the execution on the next command for a given number of seconds. +type WaitCommandSpec struct { + // Seconds to pause before next command. + // +optional + Seconds int32 `json:"seconds,omitempty"` +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..a9f858e --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the operator v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=operator.kubeadm.x-k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "operator.kubeadm.x-k8s.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/labels.go b/api/v1alpha1/labels.go new file mode 100644 index 0000000..33dcc96 --- /dev/null +++ b/api/v1alpha1/labels.go @@ -0,0 +1,36 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +const ( + // OperationNameLabel is a label defined for allowing lookup of objects related + // to one Operation. + OperationNameLabel = "operator.kubeadm.x-k8s.io/operation" + + // OperationUIDLabel is a label defined for ensuring that objects related + // to one Operation won't get mixed by chance. + OperationUIDLabel = "operator.kubeadm.x-k8s.io/uid" + + // TaskGroupNameLabel is a label defined for allowing lookup of objects related + // to one TaskGroup object. + TaskGroupNameLabel = "operator.kubeadm.x-k8s.io/taskgroup" + + // TaskGroupOrderLabel is a label defined allowing lookup of objects related + // to one TaskGroup object by the TaskGroup sequential order + // e.g. list of the Task related to the first TaskGroup. + TaskGroupOrderLabel = "operator.kubeadm.x-k8s.io/order" +) diff --git a/api/v1alpha1/operation_descriptor_types.go b/api/v1alpha1/operation_descriptor_types.go new file mode 100644 index 0000000..e8e9902 --- /dev/null +++ b/api/v1alpha1/operation_descriptor_types.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OperatorDescriptor represents an operation to be performed. +// Only one of its members may be specified. +type OperatorDescriptor struct { + // Upgrade provide declarative support for the kubeadm upgrade workflow. + // +optional + Upgrade *UpgradeOperationSpec `json:"upgrade,omitempty"` + + // RenewCertificates provide declarative support for the kubeadm upgrade workflow. + // +optional + RenewCertificates *RenewCertificatesOperationSpec `json:"renewCertificates,omitempty"` + + // CustomOperation enable definition of custom list of RuntimeTaskGroup. + // +optional + CustomOperation *CustomOperationSpec `json:"custom,omitempty"` +} + +// UpgradeOperationSpec provide declarative support for the kubeadm upgrade workflow. +type UpgradeOperationSpec struct { + // KubernetesVersion specifies the target kubernetes version + KubernetesVersion string `json:"kubernetesVersion"` + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// RenewCertificatesOperationSpec provide declarative support for the kubeadm upgrade workflow. +type RenewCertificatesOperationSpec struct { + + // INSERT ADDITIONAL SPEC FIELDS - + // Important: Run "make" to regenerate code after modifying this file +} + +// CustomOperationSpec enable definition of custom list of RuntimeTaskGroup. +type CustomOperationSpec struct { + // Workflow allows to define a custom list of RuntimeTaskGroup. + Workflow []RuntimeTaskGroup `json:"workflow"` +} diff --git a/api/v1alpha1/operation_execution_mode.go b/api/v1alpha1/operation_execution_mode.go new file mode 100644 index 0000000..03147b4 --- /dev/null +++ b/api/v1alpha1/operation_execution_mode.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OperationExecutionMode is a string representation of a RuntimeTaskGroup create policy. +type OperationExecutionMode string + +const ( + // OperationExecutionModeAuto forces the kubeadm operator to automatically execute a new RuntimeTask/Command immediately + // after the current RuntimeTask/Command is completed successfully. + OperationExecutionModeAuto = OperationExecutionMode("Auto") + + // OperationExecutionModeControlled forces the kubeadm operator to pause immediately before executing a new + // RuntimeTask/Command, so the user can check the RuntimeTask specification and decide if to proceed or not. + OperationExecutionModeControlled = OperationExecutionMode("Controlled") + + // OperationExecutionModeDryRun forces the kubeadm operator to dry-run instead of actually executing RuntimeTasks/Commands. + OperationExecutionModeDryRun = OperationExecutionMode("DryRun") + + // OperationExecutionModeUnknown is returned if the OperationExecutionMode cannot be determined. + OperationExecutionModeUnknown = OperationExecutionMode("") +) + +// GetTypedOperationExecutionMode attempts to parse the ExecutionMode field and return +// the typed OperationExecutionMode representation. +func (s *OperationSpec) GetTypedOperationExecutionMode() OperationExecutionMode { + switch mode := OperationExecutionMode(s.ExecutionMode); mode { + case + OperationExecutionModeAuto, + OperationExecutionModeDryRun, + OperationExecutionModeControlled: + return mode + default: + return OperationExecutionModeUnknown + } +} diff --git a/api/v1alpha1/operation_types.go b/api/v1alpha1/operation_types.go new file mode 100644 index 0000000..a41b10a --- /dev/null +++ b/api/v1alpha1/operation_types.go @@ -0,0 +1,158 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + operatorerrors "k8s.io/kubeadm/operator/errors" +) + +// OperationSpec defines the spec of an Operation to be performed by the kubeadm-operator. +// Please note that once the operation will be completed, the operator will stop to reconcile on any instance of this object. +type OperationSpec struct { + // Paused indicates that the operation is paused. + // +optional + Paused bool `json:"paused,omitempty"` + + // ExecutionMode indicates how the controller should handle RuntimeTask/Command execution for this operation. + // If missing, auto mode will be used. + // +optional + ExecutionMode string `json:"executionMode,omitempty"` + + // OperatorDescriptor provide description of the operator content + OperatorDescriptor `json:",inline"` +} + +// OperationStatus defines the observed state of Operation +type OperationStatus struct { + + // StartTime represents time when the Operation execution was started by the controller. + // It is represented in RFC3339 form and is in UTC. + // +optional + StartTime *metav1.Time `json:"startTime,omitempty"` + + // Paused indicates that the Operation is paused. + // This fields is set when the OperationSpec.Paused value is processed by the controller. + // +optional + Paused bool `json:"paused,omitempty"` + + // CompletionTime represents time when the Operation was completed. + // It is represented in RFC3339 form and is in UTC. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Groups returns the number of RuntimeTaskGroups belonging to this operation. + // +optional + Groups int32 `json:"groups,omitempty"` + + // RunningGroups + // +optional + RunningGroups int32 `json:"runningGroups,omitempty"` + + // SucceededGroups return the number of RuntimeTaskGroups where all the RuntimeTask succeeded + // +optional + SucceededGroups int32 `json:"succeededGroups,omitempty"` + + // FailedGroups return the number of RuntimeTaskGroups with at least one RuntimeTask failure + // +optional + FailedGroups int32 `json:"failedGroups,omitempty"` + + // InvalidGroups return the number of RuntimeTaskGroups with at least one RuntimeTask inconsistencies like e.g. orphans + // +optional + InvalidGroups int32 `json:"invalidGroups,omitempty"` + + // Phase represents the current phase of Operation actuation. + // E.g. pending, running, succeeded, failed etc. + // +optional + Phase string `json:"phase,omitempty"` + + // ErrorReason will be set in the event that there is a problem in executing + // one of the Operation's RuntimeTasks and will contain a succinct value suitable + // for machine interpretation. + // +optional + ErrorReason *operatorerrors.OperationStatusError `json:"errorReason,omitempty"` + + // ErrorMessage will be set in the event that there is a problem in executing + // one of the Operation's RuntimeTasks and will contain a more verbose string suitable + // for logging and human consumption. + // +optional + ErrorMessage *string `json:"errorMessage,omitempty"` +} + +// SetStartTime is a utility method for setting the StartTime field to Now. +func (s *OperationStatus) SetStartTime() { + now := metav1.Now() + s.StartTime = &now +} + +// SetCompletionTime is a utility method for setting the CompletionTime field to Now. +func (s *OperationStatus) SetCompletionTime() { + now := metav1.Now() + s.CompletionTime = &now + + // ensure all the status attributes are clean after complete + s.Paused = false + s.ErrorReason = nil + s.ErrorMessage = nil +} + +// SetError is a utility method for setting the ErrorReason and ErrorMessage fields +// given an OperationError object. +func (s *OperationStatus) SetError(err *operatorerrors.OperationError) { + reason := err.Reason + s.ErrorReason = &reason + s.ErrorMessage = pointer.StringPtr(err.Message) +} + +// ResetError is a utility method for resetting the ErrorReason and ErrorMessage fields +func (s *OperationStatus) ResetError() { + s.ErrorReason = nil + s.ErrorMessage = nil +} + +// +kubebuilder:resource:path=operations,scope=Cluster,categories=kubeadm-operator +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:object:root=true +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Groups",type="integer",JSONPath=".status.groups" +// +kubebuilder:printcolumn:name="Succeeded",type="integer",JSONPath=".status.succeededGroups" +// +kubebuilder:printcolumn:name="Failed",type="integer",JSONPath=".status.failedGroups" + +// Operation is the Schema for the operations API +type Operation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OperationSpec `json:"spec,omitempty"` + Status OperationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// OperationList contains a list of Operation +type OperationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Operation `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Operation{}, &OperationList{}) +} diff --git a/api/v1alpha1/operator_phase_types.go b/api/v1alpha1/operator_phase_types.go new file mode 100644 index 0000000..d38ef54 --- /dev/null +++ b/api/v1alpha1/operator_phase_types.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OperationPhase is a string representation of a Operation Phase. +// +// This type is a high-level indicator of the status of the Operation as it is provisioned, +// from the API user’s perspective. +// +// The value should not be interpreted by any software components as a reliable indication +// of the actual state of the Operation, and controllers should not use the Operation Phase field +// value when making decisions about what action to take. +// +// Controllers should always look at the actual state of the Operation’s fields to make those decisions. +type OperationPhase string + +const ( + // OperationPhasePending is the first state a Operation is assigned after being created. + OperationPhasePending = OperationPhase("Pending") + + // OperationPhaseRunning is the Operation state when it has + // started its actuation. + OperationPhaseRunning = OperationPhase("Running") + + // OperationPhasePaused is the Operation state when it is paused. + OperationPhasePaused = OperationPhase("Paused") + + // OperationPhaseSucceeded is the Operation state when all the + // desired RuntimeTasks are succeeded. + OperationPhaseSucceeded = OperationPhase("Succeeded") + + // OperationPhaseFailed is the Operation state when the system + // might require user intervention. + OperationPhaseFailed = OperationPhase("Failed") + + // OperationPhaseDeleted is the Operation state when the object + // is deleted and ready to be garbage collected by the API Server. + OperationPhaseDeleted = OperationPhase("Deleted") + + //OperationPhaseUnknown is returned if the Operation state cannot be determined. + OperationPhaseUnknown = OperationPhase("") +) + +// SetTypedPhase sets the Phase field to the string representation of OperationPhase. +func (s *OperationStatus) SetTypedPhase(p OperationPhase) { + s.Phase = string(p) +} + +// GetTypedPhase attempts to parse the Phase field and return +// the typed OperationPhase representation. +func (s *OperationStatus) GetTypedPhase() OperationPhase { + switch phase := OperationPhase(s.Phase); phase { + case + OperationPhasePending, + OperationPhaseRunning, + OperationPhasePaused, + OperationPhaseSucceeded, + OperationPhaseFailed, + OperationPhaseDeleted: + return phase + default: + return OperationPhaseUnknown + } +} diff --git a/api/v1alpha1/runtimetask_phase_types.go b/api/v1alpha1/runtimetask_phase_types.go new file mode 100644 index 0000000..3160a84 --- /dev/null +++ b/api/v1alpha1/runtimetask_phase_types.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// RuntimeTaskPhase is a string representation of a RuntimeTask Phase. +// +// This type is a high-level indicator of the status of the RuntimeTask as it is provisioned, +// from the API user’s perspective. +// +// The value should not be interpreted by any software components as a reliable indication +// of the actual state of the RuntimeTask, and controllers should not use the RuntimeTask Phase field +// value when making decisions about what action to take. +// +// Controllers should always look at the actual state of the RuntimeTask’s fields to make those decisions. +type RuntimeTaskPhase string + +const ( + // RuntimeTaskPhasePending is the first state a RuntimeTask is assigned by + // RuntimeTask controller after being created. + RuntimeTaskPhasePending = RuntimeTaskPhase("Pending") + + // RuntimeTaskPhaseRunning is the RuntimeTask state when it has + // started its actuation. + RuntimeTaskPhaseRunning = RuntimeTaskPhase("Running") + + // RuntimeTaskPhasePaused is the RuntimeTask state when it is paused. + RuntimeTaskPhasePaused = RuntimeTaskPhase("Paused") + + // RuntimeTaskPhaseSucceeded is the RuntimeTask state when it has + // succeeded its actuation. + RuntimeTaskPhaseSucceeded = RuntimeTaskPhase("Succeeded") + + // RuntimeTaskPhaseFailed is the RuntimeTask state when the system + // might require user intervention. + RuntimeTaskPhaseFailed = RuntimeTaskPhase("Failed") + + // RuntimeTaskPhaseDeleted is the RuntimeTask state when the object + // is deleted and ready to be garbage collected by the API Server. + RuntimeTaskPhaseDeleted = RuntimeTaskPhase("Deleted") + + //RuntimeTaskPhaseUnknown is returned if the RuntimeTask state cannot be determined. + RuntimeTaskPhaseUnknown = RuntimeTaskPhase("") +) + +// SetTypedPhase sets the Phase field to the string representation of RuntimeTaskPhase. +func (s *RuntimeTaskStatus) SetTypedPhase(p RuntimeTaskPhase) { + s.Phase = string(p) +} + +// GetTypedPhase attempts to parse the Phase field and return +// the typed RuntimeTaskPhase representation. +func (s *RuntimeTaskStatus) GetTypedPhase() RuntimeTaskPhase { + switch phase := RuntimeTaskPhase(s.Phase); phase { + case + RuntimeTaskPhasePending, + RuntimeTaskPhasePaused, + RuntimeTaskPhaseRunning, + RuntimeTaskPhaseSucceeded, + RuntimeTaskPhaseFailed, + RuntimeTaskPhaseDeleted: + return phase + default: + return RuntimeTaskPhaseUnknown + } +} diff --git a/api/v1alpha1/runtimetask_recovery_strategy.go b/api/v1alpha1/runtimetask_recovery_strategy.go new file mode 100644 index 0000000..56eb368 --- /dev/null +++ b/api/v1alpha1/runtimetask_recovery_strategy.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// RuntimeTaskRecoveryStrategy is a string representation of a RuntimeTask error recovery policy. +type RuntimeTaskRecoveryStrategy string + +const ( + // RuntimeTaskRecoverySkippingFailedCommandStrategy forces the RuntimeTask operator to skip failed RuntimeTask/Command. + RuntimeTaskRecoverySkippingFailedCommandStrategy = RuntimeTaskRecoveryStrategy("SkipFailedCommand") + + // RuntimeTaskRecoveryRetryingFailedCommandStrategy forces the RuntimeTask operator to retry the failed RuntimeTask/Command. + RuntimeTaskRecoveryRetryingFailedCommandStrategy = RuntimeTaskRecoveryStrategy("RetryFailedCommand") + + // RuntimeTaskRecoveryUnknownStrategy is returned if the RuntimeTaskErrorRecoveryStrategy cannot be determined. + RuntimeTaskRecoveryUnknownStrategy = RuntimeTaskRecoveryStrategy("") +) + +// GetTypedTaskRecoveryStrategy attempts to parse the mode field and return +// the typed RuntimeTaskRecoveryStrategy representation. +func (s *RuntimeTaskSpec) GetTypedTaskRecoveryStrategy() RuntimeTaskRecoveryStrategy { + switch mode := RuntimeTaskRecoveryStrategy(s.RecoveryMode); mode { + case + RuntimeTaskRecoverySkippingFailedCommandStrategy, + RuntimeTaskRecoveryRetryingFailedCommandStrategy: + return mode + default: + return RuntimeTaskRecoveryUnknownStrategy + } +} diff --git a/api/v1alpha1/runtimetask_types.go b/api/v1alpha1/runtimetask_types.go new file mode 100644 index 0000000..0b3a0b7 --- /dev/null +++ b/api/v1alpha1/runtimetask_types.go @@ -0,0 +1,151 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + operatorerrors "k8s.io/kubeadm/operator/errors" +) + +// RuntimeTaskSpec defines the desired state of RuntimeTask +type RuntimeTaskSpec struct { + // NodeName is a request to schedule this RuntimeTask onto a specific node. + // +optional + NodeName string `json:"nodeName,omitempty"` + + // RecoveryMode sets the strategy to use when a command is failed. + // +optional + RecoveryMode string `json:"recoveryMode,omitempty"` + + // Commands provide the list of commands to be performed when executing a RuntimeTask on a node + Commands []CommandDescriptor `json:"commands"` +} + +// RuntimeTaskStatus defines the observed state of RuntimeTask +type RuntimeTaskStatus struct { + + // StartTime represents time when the RuntimeTask execution was started by the controller. + // It is represented in RFC3339 form and is in UTC. + // +optional + StartTime *metav1.Time `json:"startTime,omitempty"` + + // CurrentCommand + // +optional + CurrentCommand int32 `json:"currentCommand,omitempty"` + + // CommandProgress + // Please note that this field is only for allowing a better visal representation of status + // +optional + CommandProgress string `json:"commandProgress,omitempty"` + + // Paused indicates that the RuntimeTask is paused. + // +optional + Paused bool `json:"paused,omitempty"` + + // CompletionTime represents time when the RuntimeTask was completed. + // It is represented in RFC3339 form and is in UTC. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Phase represents the current phase of RuntimeTask actuation. + // E.g. Pending, Running, Completed, Failed etc. + // +optional + Phase string `json:"phase,omitempty"` + + // ErrorReason will be set in the event that there is a problem in executing + // the RuntimeTasks and will contain a succinct value suitable + // for machine interpretation. + // +optional + ErrorReason *operatorerrors.RuntimeTaskStatusError `json:"errorReason,omitempty"` + + // ErrorMessage will be set in the event that there is a problem in executing + // the RuntimeTasks and will contain a more verbose string suitable + // for logging and human consumption. + // +optional + ErrorMessage *string `json:"errorMessage,omitempty"` +} + +// SetStartTime is a utility method for setting the StartTime field to Now. +func (s *RuntimeTaskStatus) SetStartTime() { + now := metav1.Now() + s.StartTime = &now +} + +// NextCurrentCommand is a utility method for setting the CurrentCommand and CommandProgress. +func (s *RuntimeTaskStatus) NextCurrentCommand(commands []CommandDescriptor) { + s.CurrentCommand = s.CurrentCommand + 1 + s.CommandProgress = fmt.Sprintf("%d/%d", s.CurrentCommand, len(commands)) +} + +// SetCompletionTime is a utility method for setting the CompletionTime field to Now. +func (s *RuntimeTaskStatus) SetCompletionTime() { + now := metav1.Now() + s.CompletionTime = &now + + // ensure all the status attributes are clean after complete + s.Paused = false + s.ResetError() +} + +// SetError is a utility method for setting the ErrorReason and ErrorMessage fields +// given a RuntimeTaskError object. +func (s *RuntimeTaskStatus) SetError(err *operatorerrors.RuntimeTaskError) { + reason := err.Reason + s.ErrorReason = &reason + s.ErrorMessage = pointer.StringPtr(err.Message) +} + +// ResetError is a utility method for cleaning the ErrorReason and ErrorMessage fields +func (s *RuntimeTaskStatus) ResetError() { + s.ErrorReason = nil + s.ErrorMessage = nil +} + +// +kubebuilder:resource:path=runtimetasks,scope=Cluster,categories=kubeadm-operator +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:object:root=true +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="StartTime",type="date",JSONPath=".status.startTime" +// +kubebuilder:printcolumn:name="Command",type="string",JSONPath=".status.commandProgress" +// +kubebuilder:printcolumn:name="CompletionTime",type="date",JSONPath=".status.completionTime" + +// RuntimeTask is the Schema for the runtimetasks API +type RuntimeTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RuntimeTaskSpec `json:"spec,omitempty"` + Status RuntimeTaskStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// RuntimeTaskList contains a list of RuntimeTask +type RuntimeTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RuntimeTask `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RuntimeTask{}, &RuntimeTaskList{}) +} diff --git a/api/v1alpha1/runtimetaskgroup_create_strategy.go b/api/v1alpha1/runtimetaskgroup_create_strategy.go new file mode 100644 index 0000000..00f45b8 --- /dev/null +++ b/api/v1alpha1/runtimetaskgroup_create_strategy.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// RuntimeTaskGroupCreatePolicy is a string representation of a RuntimeTaskGroup create policy. +type RuntimeTaskGroupCreatePolicy string + +const ( + // RuntimeTaskGroupCreateSerialStrategy forces the RuntimeTaskGroup controller to create RuntimeTasks in sequential order. + // New RuntimeTask are created only after the current RuntimeTime is completed successfully. + RuntimeTaskGroupCreateSerialStrategy = RuntimeTaskGroupCreatePolicy("Serial") + + // RuntimeTaskGroupCreateUnknownStrategy is returned if the RuntimeTaskGroupCreatePolicy cannot be determined. + RuntimeTaskGroupCreateUnknownStrategy = RuntimeTaskGroupCreatePolicy("") +) + +// GetTypedTaskGroupCreateStrategy attempts to parse the CreateStrategy field and return +// the typed RuntimeTaskGroupCreatePolicy representation. +func (s *RuntimeTaskGroupSpec) GetTypedTaskGroupCreateStrategy() RuntimeTaskGroupCreatePolicy { + switch mode := RuntimeTaskGroupCreatePolicy(s.CreateStrategy); mode { + case + RuntimeTaskGroupCreateSerialStrategy: + return mode + default: + return RuntimeTaskGroupCreateUnknownStrategy + } +} diff --git a/api/v1alpha1/runtimetaskgroup_node_filter.go b/api/v1alpha1/runtimetaskgroup_node_filter.go new file mode 100644 index 0000000..703e515 --- /dev/null +++ b/api/v1alpha1/runtimetaskgroup_node_filter.go @@ -0,0 +1,51 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// RuntimeTaskGroupNodeFilter is a string representation of a RuntimeTaskGroup node filter. +type RuntimeTaskGroupNodeFilter string + +const ( + // RuntimeTaskGroupNodeFilterAll forces the RuntimeTaskGroup controller to use all the nodes + // returned by the NodeSelector. + RuntimeTaskGroupNodeFilterAll = RuntimeTaskGroupNodeFilter("All") + + // RuntimeTaskGroupNodeFilterHead forces the RuntimeTaskGroup controller to use only the first node + // returned by the NodeSelector. + RuntimeTaskGroupNodeFilterHead = RuntimeTaskGroupNodeFilter("Head") + + // RuntimeTaskGroupNodeFilterTail forces the RuntimeTaskGroup controller to use all the nodes + // returned by the NodeSelector except the first one. + RuntimeTaskGroupNodeFilterTail = RuntimeTaskGroupNodeFilter("Tail") + + // RuntimeTaskGroupNodeUnknownFilter is returned if the RuntimeTaskGroupNodeFilter cannot be determined. + RuntimeTaskGroupNodeUnknownFilter = RuntimeTaskGroupNodeFilter("") +) + +// GetTypedTaskGroupNodeFilter attempts to parse the NodeFilter field and return +// the typed RuntimeTaskGroupNodeFilter representation. +func (s *RuntimeTaskGroupSpec) GetTypedTaskGroupNodeFilter() RuntimeTaskGroupNodeFilter { + switch mode := RuntimeTaskGroupNodeFilter(s.NodeFilter); mode { + case + RuntimeTaskGroupNodeFilterAll, + RuntimeTaskGroupNodeFilterHead, + RuntimeTaskGroupNodeFilterTail: + return mode + default: + return RuntimeTaskGroupNodeUnknownFilter + } +} diff --git a/api/v1alpha1/runtimetaskgroup_phase_types.go b/api/v1alpha1/runtimetaskgroup_phase_types.go new file mode 100644 index 0000000..5a4f7d1 --- /dev/null +++ b/api/v1alpha1/runtimetaskgroup_phase_types.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// RuntimeTaskGroupPhase is a string representation of a RuntimeTaskGroup Phase. +// +// This type is a high-level indicator of the status of the RuntimeTaskGroup as it is provisioned, +// from the API user’s perspective. +// +// The value should not be interpreted by any software components as a reliable indication +// of the actual state of the RuntimeTaskGroup, and controllers should not use the RuntimeTaskGroup Phase field +// value when making decisions about what action to take. +// +// Controllers should always look at the actual state of the RuntimeTaskGroup’s fields to make those decisions. +type RuntimeTaskGroupPhase string + +const ( + // RuntimeTaskGroupPhasePending is the first state a RuntimeTaskGroup is assigned by + // Operation controller after being created. + RuntimeTaskGroupPhasePending = RuntimeTaskGroupPhase("Pending") + + // RuntimeTaskGroupPhaseRunning is the RuntimeTaskGroup state when it has + // started its actuation. + RuntimeTaskGroupPhaseRunning = RuntimeTaskGroupPhase("Running") + + // RuntimeTaskGroupPhasePaused is the RuntimeTaskGroup state when it is paused. + RuntimeTaskGroupPhasePaused = RuntimeTaskGroupPhase("Paused") + + // RuntimeTaskGroupPhaseSucceeded is the RuntimeTaskGroup state when all the + // RuntimeTasks are succeeded. + RuntimeTaskGroupPhaseSucceeded = RuntimeTaskGroupPhase("Succeeded") + + // RuntimeTaskGroupPhaseFailed is the RuntimeTaskGroup state when the system + // might require user intervention. + RuntimeTaskGroupPhaseFailed = RuntimeTaskGroupPhase("Failed") + + // RuntimeTaskGroupPhaseDeleted is the RuntimeTaskGroup state when the object + // is deleted and ready to be garbage collected by the API Server. + RuntimeTaskGroupPhaseDeleted = RuntimeTaskGroupPhase("Deleted") + + //RuntimeTaskGroupPhaseUnknown is returned if the RuntimeTaskGroup state cannot be determined. + RuntimeTaskGroupPhaseUnknown = RuntimeTaskGroupPhase("") +) + +// SetTypedPhase sets the Phase field to the string representation of RuntimeTaskGroupPhase. +func (s *RuntimeTaskGroupStatus) SetTypedPhase(p RuntimeTaskGroupPhase) { + s.Phase = string(p) +} + +// GetTypedPhase attempts to parse the Phase field and return +// the typed RuntimeTaskGroupPhase representation. +func (s *RuntimeTaskGroupStatus) GetTypedPhase() RuntimeTaskGroupPhase { + switch phase := RuntimeTaskGroupPhase(s.Phase); phase { + case + RuntimeTaskGroupPhasePending, + RuntimeTaskGroupPhasePaused, + RuntimeTaskGroupPhaseRunning, + RuntimeTaskGroupPhaseSucceeded, + RuntimeTaskGroupPhaseFailed, + RuntimeTaskGroupPhaseDeleted: + return phase + default: + return RuntimeTaskGroupPhaseUnknown + } +} diff --git a/api/v1alpha1/runtimetaskgroup_types.go b/api/v1alpha1/runtimetaskgroup_types.go new file mode 100644 index 0000000..8510334 --- /dev/null +++ b/api/v1alpha1/runtimetaskgroup_types.go @@ -0,0 +1,190 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + operatorerrors "k8s.io/kubeadm/operator/errors" +) + +// RuntimeTaskGroupSpec defines the RuntimeTask template that applies to a group of nodes that should +// be changed as part of an Operation. +// Please note that this is a runtime object, create with the goal to allow ensure orchestration of +// operation RuntimeTasks/Commands, and that this object will be deleted immediately after the operation completes. +// Users can refer to this object only in case of errors/for problem investigation. +type RuntimeTaskGroupSpec struct { + // NodeSelector is a label query that identifies the list of nodes that should be impacted by this RuntimeTaskGroup. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + // +optional + NodeSelector metav1.LabelSelector `json:"nodeSelector,omitempty"` + + // NodeFilter allows to filter the list of nodes returned by the node selector. + // It defaults to all. + // +optional + NodeFilter string `json:"nodeFilter,omitempty"` + + // Selector is a label query over RuntimeTasks that are generated by this RuntimeTaskGroup. + // Label keys and values that must match in order to be controlled by this RuntimeTaskGroup. + // It must match the RuntimeTask template's labels. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + // +optional + Selector metav1.LabelSelector `json:"selector,omitempty"` + + // Template is the object that describes the RuntimeTask that will be created if + // insufficient replicas are detected. + // +optional + Template RuntimeTaskTemplateSpec `json:"template,omitempty"` + + // CreateStrategy indicates how the controller should handle RuntimeTask creation for this RuntimeTaskGroup. + // If missing, sequential mode will be used. + // +optional + CreateStrategy string `json:"createStrategy,omitempty"` +} + +// RuntimeTaskTemplateSpec defines the RuntimeTask that applies to a group of nodes that should +// be changed as part of an Operation. +type RuntimeTaskTemplateSpec struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the RuntimeTask. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status + // +optional + Spec RuntimeTaskSpec `json:"spec,omitempty"` +} + +// RuntimeTaskGroupStatus defines the observed state of RuntimeTaskGroup +type RuntimeTaskGroupStatus struct { + + // StartTime represents time when the RuntimeTaskGroup execution was started by the controller. + // It is represented in RFC3339 form and is in UTC. + // +optional + StartTime *metav1.Time `json:"startTime,omitempty"` + + // Paused indicates that the RuntimeTaskGroup is currently paused. + // This fields is set when the OperationSpec.Paused value is processed by the controller. + // +optional + Paused bool `json:"paused,omitempty"` + + // CompletionTime represents time when the RuntimeTaskGroup was completed. + // It is represented in RFC3339 form and is in UTC. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Nodes + // +optional + Nodes int32 `json:"nodes,omitempty"` + + // RunningNodes + // +optional + RunningNodes int32 `json:"runningNodes,omitempty"` + + // SucceededNodes + // +optional + SucceededNodes int32 `json:"succeededNodes,omitempty"` + + // FailedNodes + // +optional + FailedNodes int32 `json:"failedNodes,omitempty"` + + // InvalidNodes + // +optional + InvalidNodes int32 `json:"invalidNodes,omitempty"` + + // Phase represents the current phase of RuntimeTaskGroup actuation. + // E.g. pending, running, succeeded, failed etc. + // +optional + Phase string `json:"phase,omitempty"` + + // ErrorReason will be set in the event that there is a problem in executing + // one of the RuntimeTaskGroup's RuntimeTasks and will contain a succinct value suitable + // for machine interpretation. + // +optional + ErrorReason *operatorerrors.RuntimeTaskGroupStatusError `json:"errorReason,omitempty"` + + // ErrorMessage will be set in the event that there is a problem in executing + // one of the RuntimeTaskGroup's RuntimeTasks and will contain a more verbose string suitable + // for logging and human consumption. + // +optional + ErrorMessage *string `json:"errorMessage,omitempty"` +} + +// SetStartTime is a utility method for setting the StartTime field to Now. +func (s *RuntimeTaskGroupStatus) SetStartTime() { + now := metav1.Now() + s.StartTime = &now +} + +// SetCompletionTime is a utility method for setting the CompletionTime field to Now. +func (s *RuntimeTaskGroupStatus) SetCompletionTime() { + now := metav1.Now() + s.CompletionTime = &now + + // ensure all the status attributes are clean after complete + s.Paused = false + s.ErrorReason = nil + s.ErrorMessage = nil +} + +// SetError is a utility method for setting the ErrorReason and ErrorMessage fields +// given a RuntimeTaskGroupError object. +func (s *RuntimeTaskGroupStatus) SetError(err *operatorerrors.RuntimeTaskGroupError) { + reason := err.Reason + s.ErrorReason = &reason + s.ErrorMessage = pointer.StringPtr(err.Message) +} + +// ResetError is a utility method for resetting the ErrorReason and ErrorMessage fields +func (s *RuntimeTaskGroupStatus) ResetError() { + s.ErrorReason = nil + s.ErrorMessage = nil +} + +// +kubebuilder:resource:path=runtimetaskgroups,scope=Cluster,categories=kubeadm-operator +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:object:root=true +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Nodes",type="integer",JSONPath=".status.nodes" +// +kubebuilder:printcolumn:name="Succeeded",type="integer",JSONPath=".status.succeededNodes" +// +kubebuilder:printcolumn:name="Failed",type="integer",JSONPath=".status.failedNodes" + +// RuntimeTaskGroup is the Schema for the runtimetaskgroups API +type RuntimeTaskGroup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RuntimeTaskGroupSpec `json:"spec,omitempty"` + Status RuntimeTaskGroupStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// RuntimeTaskGroupList contains a list of RuntimeTaskGroup +type RuntimeTaskGroupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RuntimeTaskGroup `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RuntimeTaskGroup{}, &RuntimeTaskGroupList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..0d5f4c3 --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,692 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubeadm/operator/errors" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommandDescriptor) DeepCopyInto(out *CommandDescriptor) { + *out = *in + if in.KubeadmRenewCertificates != nil { + in, out := &in.KubeadmRenewCertificates, &out.KubeadmRenewCertificates + *out = new(KubeadmRenewCertsCommandSpec) + **out = **in + } + if in.KubeadmUpgradeApply != nil { + in, out := &in.KubeadmUpgradeApply, &out.KubeadmUpgradeApply + *out = new(KubeadmUpgradeApplyCommandSpec) + **out = **in + } + if in.KubeadmUpgradeNode != nil { + in, out := &in.KubeadmUpgradeNode, &out.KubeadmUpgradeNode + *out = new(KubeadmUpgradeNodeCommandSpec) + **out = **in + } + if in.Preflight != nil { + in, out := &in.Preflight, &out.Preflight + *out = new(PreflightCommandSpec) + **out = **in + } + if in.UpgradeKubeadm != nil { + in, out := &in.UpgradeKubeadm, &out.UpgradeKubeadm + *out = new(UpgradeKubeadmCommandSpec) + **out = **in + } + if in.UpgradeKubeletAndKubeactl != nil { + in, out := &in.UpgradeKubeletAndKubeactl, &out.UpgradeKubeletAndKubeactl + *out = new(UpgradeKubeletAndKubeactlCommandSpec) + **out = **in + } + if in.KubectlDrain != nil { + in, out := &in.KubectlDrain, &out.KubectlDrain + *out = new(KubectlDrainCommandSpec) + **out = **in + } + if in.KubectlUncordon != nil { + in, out := &in.KubectlUncordon, &out.KubectlUncordon + *out = new(KubectlUncordonCommandSpec) + **out = **in + } + if in.Pass != nil { + in, out := &in.Pass, &out.Pass + *out = new(PassCommandSpec) + **out = **in + } + if in.Fail != nil { + in, out := &in.Fail, &out.Fail + *out = new(FailCommandSpec) + **out = **in + } + if in.Wait != nil { + in, out := &in.Wait, &out.Wait + *out = new(WaitCommandSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommandDescriptor. +func (in *CommandDescriptor) DeepCopy() *CommandDescriptor { + if in == nil { + return nil + } + out := new(CommandDescriptor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomOperationSpec) DeepCopyInto(out *CustomOperationSpec) { + *out = *in + if in.Workflow != nil { + in, out := &in.Workflow, &out.Workflow + *out = make([]RuntimeTaskGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomOperationSpec. +func (in *CustomOperationSpec) DeepCopy() *CustomOperationSpec { + if in == nil { + return nil + } + out := new(CustomOperationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FailCommandSpec) DeepCopyInto(out *FailCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailCommandSpec. +func (in *FailCommandSpec) DeepCopy() *FailCommandSpec { + if in == nil { + return nil + } + out := new(FailCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmRenewCertsCommandSpec) DeepCopyInto(out *KubeadmRenewCertsCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmRenewCertsCommandSpec. +func (in *KubeadmRenewCertsCommandSpec) DeepCopy() *KubeadmRenewCertsCommandSpec { + if in == nil { + return nil + } + out := new(KubeadmRenewCertsCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmUpgradeApplyCommandSpec) DeepCopyInto(out *KubeadmUpgradeApplyCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmUpgradeApplyCommandSpec. +func (in *KubeadmUpgradeApplyCommandSpec) DeepCopy() *KubeadmUpgradeApplyCommandSpec { + if in == nil { + return nil + } + out := new(KubeadmUpgradeApplyCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeadmUpgradeNodeCommandSpec) DeepCopyInto(out *KubeadmUpgradeNodeCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmUpgradeNodeCommandSpec. +func (in *KubeadmUpgradeNodeCommandSpec) DeepCopy() *KubeadmUpgradeNodeCommandSpec { + if in == nil { + return nil + } + out := new(KubeadmUpgradeNodeCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubectlDrainCommandSpec) DeepCopyInto(out *KubectlDrainCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubectlDrainCommandSpec. +func (in *KubectlDrainCommandSpec) DeepCopy() *KubectlDrainCommandSpec { + if in == nil { + return nil + } + out := new(KubectlDrainCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubectlUncordonCommandSpec) DeepCopyInto(out *KubectlUncordonCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubectlUncordonCommandSpec. +func (in *KubectlUncordonCommandSpec) DeepCopy() *KubectlUncordonCommandSpec { + if in == nil { + return nil + } + out := new(KubectlUncordonCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Operation) DeepCopyInto(out *Operation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Operation. +func (in *Operation) DeepCopy() *Operation { + if in == nil { + return nil + } + out := new(Operation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Operation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperationList) DeepCopyInto(out *OperationList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Operation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperationList. +func (in *OperationList) DeepCopy() *OperationList { + if in == nil { + return nil + } + out := new(OperationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OperationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperationSpec) DeepCopyInto(out *OperationSpec) { + *out = *in + in.OperatorDescriptor.DeepCopyInto(&out.OperatorDescriptor) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperationSpec. +func (in *OperationSpec) DeepCopy() *OperationSpec { + if in == nil { + return nil + } + out := new(OperationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperationStatus) DeepCopyInto(out *OperationStatus) { + *out = *in + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.ErrorReason != nil { + in, out := &in.ErrorReason, &out.ErrorReason + *out = new(errors.OperationStatusError) + **out = **in + } + if in.ErrorMessage != nil { + in, out := &in.ErrorMessage, &out.ErrorMessage + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperationStatus. +func (in *OperationStatus) DeepCopy() *OperationStatus { + if in == nil { + return nil + } + out := new(OperationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperatorDescriptor) DeepCopyInto(out *OperatorDescriptor) { + *out = *in + if in.Upgrade != nil { + in, out := &in.Upgrade, &out.Upgrade + *out = new(UpgradeOperationSpec) + **out = **in + } + if in.RenewCertificates != nil { + in, out := &in.RenewCertificates, &out.RenewCertificates + *out = new(RenewCertificatesOperationSpec) + **out = **in + } + if in.CustomOperation != nil { + in, out := &in.CustomOperation, &out.CustomOperation + *out = new(CustomOperationSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorDescriptor. +func (in *OperatorDescriptor) DeepCopy() *OperatorDescriptor { + if in == nil { + return nil + } + out := new(OperatorDescriptor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PassCommandSpec) DeepCopyInto(out *PassCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PassCommandSpec. +func (in *PassCommandSpec) DeepCopy() *PassCommandSpec { + if in == nil { + return nil + } + out := new(PassCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreflightCommandSpec) DeepCopyInto(out *PreflightCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreflightCommandSpec. +func (in *PreflightCommandSpec) DeepCopy() *PreflightCommandSpec { + if in == nil { + return nil + } + out := new(PreflightCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RenewCertificatesOperationSpec) DeepCopyInto(out *RenewCertificatesOperationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RenewCertificatesOperationSpec. +func (in *RenewCertificatesOperationSpec) DeepCopy() *RenewCertificatesOperationSpec { + if in == nil { + return nil + } + out := new(RenewCertificatesOperationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTask) DeepCopyInto(out *RuntimeTask) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTask. +func (in *RuntimeTask) DeepCopy() *RuntimeTask { + if in == nil { + return nil + } + out := new(RuntimeTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RuntimeTask) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskGroup) DeepCopyInto(out *RuntimeTaskGroup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskGroup. +func (in *RuntimeTaskGroup) DeepCopy() *RuntimeTaskGroup { + if in == nil { + return nil + } + out := new(RuntimeTaskGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RuntimeTaskGroup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskGroupList) DeepCopyInto(out *RuntimeTaskGroupList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RuntimeTaskGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskGroupList. +func (in *RuntimeTaskGroupList) DeepCopy() *RuntimeTaskGroupList { + if in == nil { + return nil + } + out := new(RuntimeTaskGroupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RuntimeTaskGroupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskGroupSpec) DeepCopyInto(out *RuntimeTaskGroupSpec) { + *out = *in + in.NodeSelector.DeepCopyInto(&out.NodeSelector) + in.Selector.DeepCopyInto(&out.Selector) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskGroupSpec. +func (in *RuntimeTaskGroupSpec) DeepCopy() *RuntimeTaskGroupSpec { + if in == nil { + return nil + } + out := new(RuntimeTaskGroupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskGroupStatus) DeepCopyInto(out *RuntimeTaskGroupStatus) { + *out = *in + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.ErrorReason != nil { + in, out := &in.ErrorReason, &out.ErrorReason + *out = new(errors.RuntimeTaskGroupStatusError) + **out = **in + } + if in.ErrorMessage != nil { + in, out := &in.ErrorMessage, &out.ErrorMessage + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskGroupStatus. +func (in *RuntimeTaskGroupStatus) DeepCopy() *RuntimeTaskGroupStatus { + if in == nil { + return nil + } + out := new(RuntimeTaskGroupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskList) DeepCopyInto(out *RuntimeTaskList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RuntimeTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskList. +func (in *RuntimeTaskList) DeepCopy() *RuntimeTaskList { + if in == nil { + return nil + } + out := new(RuntimeTaskList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RuntimeTaskList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskSpec) DeepCopyInto(out *RuntimeTaskSpec) { + *out = *in + if in.Commands != nil { + in, out := &in.Commands, &out.Commands + *out = make([]CommandDescriptor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskSpec. +func (in *RuntimeTaskSpec) DeepCopy() *RuntimeTaskSpec { + if in == nil { + return nil + } + out := new(RuntimeTaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskStatus) DeepCopyInto(out *RuntimeTaskStatus) { + *out = *in + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.ErrorReason != nil { + in, out := &in.ErrorReason, &out.ErrorReason + *out = new(errors.RuntimeTaskStatusError) + **out = **in + } + if in.ErrorMessage != nil { + in, out := &in.ErrorMessage, &out.ErrorMessage + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskStatus. +func (in *RuntimeTaskStatus) DeepCopy() *RuntimeTaskStatus { + if in == nil { + return nil + } + out := new(RuntimeTaskStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeTaskTemplateSpec) DeepCopyInto(out *RuntimeTaskTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeTaskTemplateSpec. +func (in *RuntimeTaskTemplateSpec) DeepCopy() *RuntimeTaskTemplateSpec { + if in == nil { + return nil + } + out := new(RuntimeTaskTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpgradeKubeadmCommandSpec) DeepCopyInto(out *UpgradeKubeadmCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradeKubeadmCommandSpec. +func (in *UpgradeKubeadmCommandSpec) DeepCopy() *UpgradeKubeadmCommandSpec { + if in == nil { + return nil + } + out := new(UpgradeKubeadmCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpgradeKubeletAndKubeactlCommandSpec) DeepCopyInto(out *UpgradeKubeletAndKubeactlCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradeKubeletAndKubeactlCommandSpec. +func (in *UpgradeKubeletAndKubeactlCommandSpec) DeepCopy() *UpgradeKubeletAndKubeactlCommandSpec { + if in == nil { + return nil + } + out := new(UpgradeKubeletAndKubeactlCommandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpgradeOperationSpec) DeepCopyInto(out *UpgradeOperationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradeOperationSpec. +func (in *UpgradeOperationSpec) DeepCopy() *UpgradeOperationSpec { + if in == nil { + return nil + } + out := new(UpgradeOperationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WaitCommandSpec) DeepCopyInto(out *WaitCommandSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaitCommandSpec. +func (in *WaitCommandSpec) DeepCopy() *WaitCommandSpec { + if in == nil { + return nil + } + out := new(WaitCommandSpec) + in.DeepCopyInto(out) + return out +} diff --git a/commands/exec.go b/commands/exec.go new file mode 100644 index 0000000..c5ef1fe --- /dev/null +++ b/commands/exec.go @@ -0,0 +1,85 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "bufio" + "bytes" + "io" + "os" + "os/exec" +) + +type cmd struct { + command string + args []string + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +func newCmd(command string, args ...string) *cmd { + return &cmd{ + command: command, + args: args, + } +} + +func (c *cmd) Run() error { + return c.runInnnerCommand() +} + +func (c *cmd) RunWithEcho() error { + c.stdout = os.Stderr + c.stderr = os.Stdout + return c.runInnnerCommand() +} + +func (c *cmd) RunAndCapture() (lines []string, err error) { + var buff bytes.Buffer + c.stdout = &buff + c.stderr = &buff + err = c.runInnnerCommand() + + scanner := bufio.NewScanner(&buff) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + + } + return lines, err +} + +func (c *cmd) Stdin(in io.Reader) *cmd { + c.stdin = in + return c +} + +func (c *cmd) runInnnerCommand() error { + cmd := exec.Command(c.command, c.args...) + + if c.stdin != nil { + cmd.Stdin = c.stdin + } + if c.stdout != nil { + cmd.Stdout = c.stdout + } + if c.stderr != nil { + cmd.Stderr = c.stderr + } + + return cmd.Run() +} diff --git a/commands/factory.go b/commands/factory.go new file mode 100644 index 0000000..5979649 --- /dev/null +++ b/commands/factory.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "time" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +// RunCommand run the command on a node +func RunCommand(c *operatorv1.CommandDescriptor, log logr.Logger) error { + + if c.Preflight != nil { + return runPreflight(c.Preflight, log) + } + + if c.KubeadmRenewCertificates != nil { + return runKubeadmRenewCertificates(c.KubeadmRenewCertificates, log) + } + + if c.KubeadmUpgradeApply != nil { + return runKubeadmUpgradeApply(c.KubeadmUpgradeApply, log) + } + + if c.KubeadmUpgradeNode != nil { + return runKubeadmUpgradeNode(c.KubeadmUpgradeNode, log) + } + + if c.KubectlDrain != nil { + return runKubectlDrain(c.KubectlDrain, log) + } + + if c.KubectlUncordon != nil { + return runKubectlUncordon(c.KubectlUncordon, log) + } + + if c.UpgradeKubeadm != nil { + return runUpgradeKubeadm(c.UpgradeKubeadm, log) + } + + if c.UpgradeKubeletAndKubeactl != nil { + return runUpgradeKubectlAndKubelet(c.UpgradeKubeletAndKubeactl, log) + } + + if c.Pass != nil { + return nil + } + + if c.Fail != nil { + time.Sleep(5 * time.Second) + return errors.New("command fail failed") + } + + if c.Wait != nil { + time.Sleep(time.Duration(c.Wait.Seconds) * time.Second) + return nil + } + + return errors.New("invalid Task.Spec.[]CommandDescriptor. There are no command implementations matching this spec") +} diff --git a/commands/kubeadm_renew_certificate.go b/commands/kubeadm_renew_certificate.go new file mode 100644 index 0000000..21dc0da --- /dev/null +++ b/commands/kubeadm_renew_certificate.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runKubeadmRenewCertificates(spec *operatorv1.KubeadmRenewCertsCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/kubeadm_upgrade_apply.go b/commands/kubeadm_upgrade_apply.go new file mode 100644 index 0000000..630e67e --- /dev/null +++ b/commands/kubeadm_upgrade_apply.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runKubeadmUpgradeApply(spec *operatorv1.KubeadmUpgradeApplyCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/kubeadm_upgrade_node.go b/commands/kubeadm_upgrade_node.go new file mode 100644 index 0000000..caefe67 --- /dev/null +++ b/commands/kubeadm_upgrade_node.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runKubeadmUpgradeNode(spec *operatorv1.KubeadmUpgradeNodeCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/kubectl_drain.go b/commands/kubectl_drain.go new file mode 100644 index 0000000..54db2f6 --- /dev/null +++ b/commands/kubectl_drain.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runKubectlDrain(spec *operatorv1.KubectlDrainCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/kubectl_uncordon.go b/commands/kubectl_uncordon.go new file mode 100644 index 0000000..82af0c5 --- /dev/null +++ b/commands/kubectl_uncordon.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runKubectlUncordon(spec *operatorv1.KubectlUncordonCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/preflight.go b/commands/preflight.go new file mode 100644 index 0000000..517b34e --- /dev/null +++ b/commands/preflight.go @@ -0,0 +1,40 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "fmt" + "strings" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runPreflight(spec *operatorv1.PreflightCommandSpec, log logr.Logger) error { + cmd := newCmd("kubeadm", "version") + + lines, err := cmd.RunAndCapture() + if err != nil { + return errors.WithStack(err) + } + + log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n"))) + + return nil +} diff --git a/commands/upgrade_kubeadm.go b/commands/upgrade_kubeadm.go new file mode 100644 index 0000000..74bf090 --- /dev/null +++ b/commands/upgrade_kubeadm.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runUpgradeKubeadm(spec *operatorv1.UpgradeKubeadmCommandSpec, log logr.Logger) error { + return nil +} diff --git a/commands/upgrade_kubectlkubelet.go b/commands/upgrade_kubectlkubelet.go new file mode 100644 index 0000000..40145a7 --- /dev/null +++ b/commands/upgrade_kubectlkubelet.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "github.com/go-logr/logr" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func runUpgradeKubectlAndKubelet(spec *operatorv1.UpgradeKubeletAndKubeactlCommandSpec, log logr.Logger) error { + return nil +} diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 0000000..02bdc1b --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,24 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + commonName: $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..55c1246 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: certmanager.k8s.io + fieldSpecs: + - kind: Certificate + group: certmanager.k8s.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: certmanager.k8s.io + path: spec/commonName +- kind: Certificate + group: certmanager.k8s.io + path: spec/dnsNames diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml new file mode 100644 index 0000000..5f59a43 --- /dev/null +++ b/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml @@ -0,0 +1,438 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: operations.operator.kubeadm.x-k8s.io +spec: + group: operator.kubeadm.x-k8s.io + names: + categories: + - kubeadm-operator + kind: Operation + listKind: OperationList + plural: operations + singular: operation + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.groups + name: Groups + type: integer + - jsonPath: .status.succeededGroups + name: Succeeded + type: integer + - jsonPath: .status.failedGroups + name: Failed + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Operation is the Schema for the operations API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OperationSpec defines the spec of an Operation to be performed + by the kubeadm-operator. Please note that once the operation will be + completed, the operator will stop to reconcile on any instance of this + object. + properties: + custom: + description: CustomOperation enable definition of custom list of RuntimeTaskGroup. + properties: + workflow: + description: Workflow allows to define a custom list of RuntimeTaskGroup. + items: + description: RuntimeTaskGroup is the Schema for the runtimetaskgroups + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RuntimeTaskGroupSpec defines the RuntimeTask + template that applies to a group of nodes that should + be changed as part of an Operation. Please note that this + is a runtime object, create with the goal to allow ensure + orchestration of operation RuntimeTasks/Commands, and + that this object will be deleted immediately after the + operation completes. Users can refer to this object only + in case of errors/for problem investigation. + properties: + createStrategy: + description: CreateStrategy indicates how the controller + should handle RuntimeTask creation for this RuntimeTaskGroup. + If missing, sequential mode will be used. + type: string + nodeFilter: + description: NodeFilter allows to filter the list of + nodes returned by the node selector. It defaults to + all. + type: string + nodeSelector: + description: 'NodeSelector is a label query that identifies + the list of nodes that should be impacted by this + RuntimeTaskGroup. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + selector: + description: 'Selector is a label query over RuntimeTasks + that are generated by this RuntimeTaskGroup. Label + keys and values that must match in order to be controlled + by this RuntimeTaskGroup. It must match the RuntimeTask + template''s labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + template: + description: Template is the object that describes the + RuntimeTask that will be created if insufficient replicas + are detected. + properties: + metadata: + description: 'Standard object''s metadata. More + info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + type: object + spec: + description: 'Specification of the desired behavior + of the RuntimeTask. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' + properties: + commands: + description: Commands provide the list of commands + to be performed when executing a RuntimeTask + on a node + items: + description: CommandDescriptor represents + a command to be performed. Only one of its + members may be specified. + properties: + fail: + description: Fail provide a dummy command + for testing the kubeadm-operator workflow. + type: object + kubeadmRenewCertificates: + description: KubeadmRenewCertsCommandSpec + provides... + type: object + kubeadmUpgradeApply: + description: KubeadmUpgradeApplyCommandSpec + provides... + type: object + kubeadmUpgradeNode: + description: KubeadmUpgradeNodeCommandSpec + provides... + type: object + kubectlDrain: + description: KubectlDrainCommandSpec provides... + type: object + kubectlUncordon: + description: KubectlUncordonCommandSpec + provides... + type: object + pass: + description: Pass provide a dummy command + for testing the kubeadm-operator workflow. + type: object + preflight: + description: PreflightCommandSpec provides... + type: object + upgradeKubeadm: + description: UpgradeKubeadmCommandSpec + provides... + type: object + upgradeKubeletAndKubeactl: + description: UpgradeKubeletAndKubeactlCommandSpec + provides... + type: object + wait: + description: Wait pauses the execution + on the next command for a given number + of seconds. + properties: + seconds: + description: Seconds to pause before + next command. + format: int32 + type: integer + type: object + type: object + type: array + nodeName: + description: NodeName is a request to schedule + this RuntimeTask onto a specific node. + type: string + recoveryMode: + description: RecoveryMode sets the strategy + to use when a command is failed. + type: string + required: + - commands + type: object + type: object + type: object + status: + description: RuntimeTaskGroupStatus defines the observed + state of RuntimeTaskGroup + properties: + completionTime: + description: CompletionTime represents time when the + RuntimeTaskGroup was completed. It is represented + in RFC3339 form and is in UTC. + format: date-time + type: string + errorMessage: + description: ErrorMessage will be set in the event that + there is a problem in executing one of the RuntimeTaskGroup's + RuntimeTasks and will contain a more verbose string + suitable for logging and human consumption. + type: string + errorReason: + description: ErrorReason will be set in the event that + there is a problem in executing one of the RuntimeTaskGroup's + RuntimeTasks and will contain a succinct value suitable + for machine interpretation. + type: string + failedNodes: + description: FailedNodes + format: int32 + type: integer + invalidNodes: + description: InvalidNodes + format: int32 + type: integer + nodes: + description: Nodes + format: int32 + type: integer + paused: + description: Paused indicates that the RuntimeTaskGroup + is currently paused. This fields is set when the OperationSpec.Paused + value is processed by the controller. + type: boolean + phase: + description: Phase represents the current phase of RuntimeTaskGroup + actuation. E.g. pending, running, succeeded, failed + etc. + type: string + runningNodes: + description: RunningNodes + format: int32 + type: integer + startTime: + description: StartTime represents time when the RuntimeTaskGroup + execution was started by the controller. It is represented + in RFC3339 form and is in UTC. + format: date-time + type: string + succeededNodes: + description: SucceededNodes + format: int32 + type: integer + type: object + type: object + type: array + required: + - workflow + type: object + executionMode: + description: ExecutionMode indicates how the controller should handle + RuntimeTask/Command execution for this operation. If missing, auto + mode will be used. + type: string + paused: + description: Paused indicates that the operation is paused. + type: boolean + renewCertificates: + description: RenewCertificates provide declarative support for the + kubeadm upgrade workflow. + type: object + upgrade: + description: Upgrade provide declarative support for the kubeadm upgrade + workflow. + properties: + kubernetesVersion: + description: KubernetesVersion specifies the target kubernetes + version + type: string + required: + - kubernetesVersion + type: object + type: object + status: + description: OperationStatus defines the observed state of Operation + properties: + completionTime: + description: CompletionTime represents time when the Operation was + completed. It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + errorMessage: + description: ErrorMessage will be set in the event that there is a + problem in executing one of the Operation's RuntimeTasks and will + contain a more verbose string suitable for logging and human consumption. + type: string + errorReason: + description: ErrorReason will be set in the event that there is a + problem in executing one of the Operation's RuntimeTasks and will + contain a succinct value suitable for machine interpretation. + type: string + failedGroups: + description: FailedGroups return the number of RuntimeTaskGroups with + at least one RuntimeTask failure + format: int32 + type: integer + groups: + description: Groups returns the number of RuntimeTaskGroups belonging + to this operation. + format: int32 + type: integer + invalidGroups: + description: InvalidGroups return the number of RuntimeTaskGroups + with at least one RuntimeTask inconsistencies like e.g. orphans + format: int32 + type: integer + paused: + description: Paused indicates that the Operation is paused. This fields + is set when the OperationSpec.Paused value is processed by the controller. + type: boolean + phase: + description: Phase represents the current phase of Operation actuation. + E.g. pending, running, succeeded, failed etc. + type: string + runningGroups: + description: RunningGroups + format: int32 + type: integer + startTime: + description: StartTime represents time when the Operation execution + was started by the controller. It is represented in RFC3339 form + and is in UTC. + format: date-time + type: string + succeededGroups: + description: SucceededGroups return the number of RuntimeTaskGroups + where all the RuntimeTask succeeded + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml new file mode 100644 index 0000000..6907b6f --- /dev/null +++ b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml @@ -0,0 +1,299 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: runtimetaskgroups.operator.kubeadm.x-k8s.io +spec: + group: operator.kubeadm.x-k8s.io + names: + categories: + - kubeadm-operator + kind: RuntimeTaskGroup + listKind: RuntimeTaskGroupList + plural: runtimetaskgroups + singular: runtimetaskgroup + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.nodes + name: Nodes + type: integer + - jsonPath: .status.succeededNodes + name: Succeeded + type: integer + - jsonPath: .status.failedNodes + name: Failed + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: RuntimeTaskGroup is the Schema for the runtimetaskgroups API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RuntimeTaskGroupSpec defines the RuntimeTask template that + applies to a group of nodes that should be changed as part of an Operation. + Please note that this is a runtime object, create with the goal to allow + ensure orchestration of operation RuntimeTasks/Commands, and that this + object will be deleted immediately after the operation completes. Users + can refer to this object only in case of errors/for problem investigation. + properties: + createStrategy: + description: CreateStrategy indicates how the controller should handle + RuntimeTask creation for this RuntimeTaskGroup. If missing, sequential + mode will be used. + type: string + nodeFilter: + description: NodeFilter allows to filter the list of nodes returned + by the node selector. It defaults to all. + type: string + nodeSelector: + description: 'NodeSelector is a label query that identifies the list + of nodes that should be impacted by this RuntimeTaskGroup. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + selector: + description: 'Selector is a label query over RuntimeTasks that are + generated by this RuntimeTaskGroup. Label keys and values that must + match in order to be controlled by this RuntimeTaskGroup. It must + match the RuntimeTask template''s labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + template: + description: Template is the object that describes the RuntimeTask + that will be created if insufficient replicas are detected. + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + type: object + spec: + description: 'Specification of the desired behavior of the RuntimeTask. + More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' + properties: + commands: + description: Commands provide the list of commands to be performed + when executing a RuntimeTask on a node + items: + description: CommandDescriptor represents a command to be + performed. Only one of its members may be specified. + properties: + fail: + description: Fail provide a dummy command for testing + the kubeadm-operator workflow. + type: object + kubeadmRenewCertificates: + description: KubeadmRenewCertsCommandSpec provides... + type: object + kubeadmUpgradeApply: + description: KubeadmUpgradeApplyCommandSpec provides... + type: object + kubeadmUpgradeNode: + description: KubeadmUpgradeNodeCommandSpec provides... + type: object + kubectlDrain: + description: KubectlDrainCommandSpec provides... + type: object + kubectlUncordon: + description: KubectlUncordonCommandSpec provides... + type: object + pass: + description: Pass provide a dummy command for testing + the kubeadm-operator workflow. + type: object + preflight: + description: PreflightCommandSpec provides... + type: object + upgradeKubeadm: + description: UpgradeKubeadmCommandSpec provides... + type: object + upgradeKubeletAndKubeactl: + description: UpgradeKubeletAndKubeactlCommandSpec provides... + type: object + wait: + description: Wait pauses the execution on the next command + for a given number of seconds. + properties: + seconds: + description: Seconds to pause before next command. + format: int32 + type: integer + type: object + type: object + type: array + nodeName: + description: NodeName is a request to schedule this RuntimeTask + onto a specific node. + type: string + recoveryMode: + description: RecoveryMode sets the strategy to use when a + command is failed. + type: string + required: + - commands + type: object + type: object + type: object + status: + description: RuntimeTaskGroupStatus defines the observed state of RuntimeTaskGroup + properties: + completionTime: + description: CompletionTime represents time when the RuntimeTaskGroup + was completed. It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + errorMessage: + description: ErrorMessage will be set in the event that there is a + problem in executing one of the RuntimeTaskGroup's RuntimeTasks + and will contain a more verbose string suitable for logging and + human consumption. + type: string + errorReason: + description: ErrorReason will be set in the event that there is a + problem in executing one of the RuntimeTaskGroup's RuntimeTasks + and will contain a succinct value suitable for machine interpretation. + type: string + failedNodes: + description: FailedNodes + format: int32 + type: integer + invalidNodes: + description: InvalidNodes + format: int32 + type: integer + nodes: + description: Nodes + format: int32 + type: integer + paused: + description: Paused indicates that the RuntimeTaskGroup is currently + paused. This fields is set when the OperationSpec.Paused value is + processed by the controller. + type: boolean + phase: + description: Phase represents the current phase of RuntimeTaskGroup + actuation. E.g. pending, running, succeeded, failed etc. + type: string + runningNodes: + description: RunningNodes + format: int32 + type: integer + startTime: + description: StartTime represents time when the RuntimeTaskGroup execution + was started by the controller. It is represented in RFC3339 form + and is in UTC. + format: date-time + type: string + succeededNodes: + description: SucceededNodes + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml new file mode 100644 index 0000000..0c2e0e9 --- /dev/null +++ b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml @@ -0,0 +1,164 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: runtimetasks.operator.kubeadm.x-k8s.io +spec: + group: operator.kubeadm.x-k8s.io + names: + categories: + - kubeadm-operator + kind: RuntimeTask + listKind: RuntimeTaskList + plural: runtimetasks + singular: runtimetask + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.startTime + name: StartTime + type: date + - jsonPath: .status.commandProgress + name: Command + type: string + - jsonPath: .status.completionTime + name: CompletionTime + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: RuntimeTask is the Schema for the runtimetasks API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RuntimeTaskSpec defines the desired state of RuntimeTask + properties: + commands: + description: Commands provide the list of commands to be performed + when executing a RuntimeTask on a node + items: + description: CommandDescriptor represents a command to be performed. + Only one of its members may be specified. + properties: + fail: + description: Fail provide a dummy command for testing the kubeadm-operator + workflow. + type: object + kubeadmRenewCertificates: + description: KubeadmRenewCertsCommandSpec provides... + type: object + kubeadmUpgradeApply: + description: KubeadmUpgradeApplyCommandSpec provides... + type: object + kubeadmUpgradeNode: + description: KubeadmUpgradeNodeCommandSpec provides... + type: object + kubectlDrain: + description: KubectlDrainCommandSpec provides... + type: object + kubectlUncordon: + description: KubectlUncordonCommandSpec provides... + type: object + pass: + description: Pass provide a dummy command for testing the kubeadm-operator + workflow. + type: object + preflight: + description: PreflightCommandSpec provides... + type: object + upgradeKubeadm: + description: UpgradeKubeadmCommandSpec provides... + type: object + upgradeKubeletAndKubeactl: + description: UpgradeKubeletAndKubeactlCommandSpec provides... + type: object + wait: + description: Wait pauses the execution on the next command for + a given number of seconds. + properties: + seconds: + description: Seconds to pause before next command. + format: int32 + type: integer + type: object + type: object + type: array + nodeName: + description: NodeName is a request to schedule this RuntimeTask onto + a specific node. + type: string + recoveryMode: + description: RecoveryMode sets the strategy to use when a command + is failed. + type: string + required: + - commands + type: object + status: + description: RuntimeTaskStatus defines the observed state of RuntimeTask + properties: + commandProgress: + description: CommandProgress Please note that this field is only for + allowing a better visal representation of status + type: string + completionTime: + description: CompletionTime represents time when the RuntimeTask was + completed. It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + currentCommand: + description: CurrentCommand + format: int32 + type: integer + errorMessage: + description: ErrorMessage will be set in the event that there is a + problem in executing the RuntimeTasks and will contain a more verbose + string suitable for logging and human consumption. + type: string + errorReason: + description: ErrorReason will be set in the event that there is a + problem in executing the RuntimeTasks and will contain a succinct + value suitable for machine interpretation. + type: string + paused: + description: Paused indicates that the RuntimeTask is paused. + type: boolean + phase: + description: Phase represents the current phase of RuntimeTask actuation. + E.g. Pending, Running, Completed, Failed etc. + type: string + startTime: + description: StartTime represents time when the RuntimeTask execution + was started by the controller. It is represented in RFC3339 form + and is in UTC. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..bdd7c9b --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,27 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/operator.kubeadm.x-k8s.io_operations.yaml +- bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml +- bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_operations.yaml +#- patches/webhook_in_runtimetaskgroups.yaml +#- patches/webhook_in_runtimetasks.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_operations.yaml +#- patches/cainjection_in_runtimetaskgroups.yaml +#- patches/cainjection_in_runtimetasks.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..8e2d8d6 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,17 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_operations.yaml b/config/crd/patches/cainjection_in_operations.yaml new file mode 100644 index 0000000..4901db6 --- /dev/null +++ b/config/crd/patches/cainjection_in_operations.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: operations.operator.kubeadm.x-k8s.io diff --git a/config/crd/patches/cainjection_in_runtimetaskgroups.yaml b/config/crd/patches/cainjection_in_runtimetaskgroups.yaml new file mode 100644 index 0000000..bca22d5 --- /dev/null +++ b/config/crd/patches/cainjection_in_runtimetaskgroups.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: runtimetaskgroups.operator.kubeadm.x-k8s.io diff --git a/config/crd/patches/cainjection_in_runtimetasks.yaml b/config/crd/patches/cainjection_in_runtimetasks.yaml new file mode 100644 index 0000000..17509f3 --- /dev/null +++ b/config/crd/patches/cainjection_in_runtimetasks.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: runtimetasks.operator.kubeadm.x-k8s.io diff --git a/config/crd/patches/webhook_in_operations.yaml b/config/crd/patches/webhook_in_operations.yaml new file mode 100644 index 0000000..2efaf67 --- /dev/null +++ b/config/crd/patches/webhook_in_operations.yaml @@ -0,0 +1,18 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: operations.operator.kubeadm.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/crd/patches/webhook_in_runtimetaskgroups.yaml b/config/crd/patches/webhook_in_runtimetaskgroups.yaml new file mode 100644 index 0000000..6485a42 --- /dev/null +++ b/config/crd/patches/webhook_in_runtimetaskgroups.yaml @@ -0,0 +1,18 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: runtimetaskgroups.operator.kubeadm.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/crd/patches/webhook_in_runtimetasks.yaml b/config/crd/patches/webhook_in_runtimetasks.yaml new file mode 100644 index 0000000..f4758ce --- /dev/null +++ b/config/crd/patches/webhook_in_runtimetasks.yaml @@ -0,0 +1,18 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: runtimetasks.operator.kubeadm.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..806dadd --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,72 @@ +# Adds namespace to all resources. +namespace: operator-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager + +patchesStrategicMerge: + # Protect the /metrics endpoint by putting it behind auth. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +- manager_auth_proxy_patch.yaml + # If you want your controller-manager to expose the /metrics + # endpoint w/o any authn/z, uncomment the following line and + # comment manager_auth_proxy_patch.yaml. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +#- manager_prometheus_metrics_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: certmanager.k8s.io +# version: v1alpha1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: certmanager.k8s.io +# version: v1alpha1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..6cb8059 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,27 @@ +# This patch inject a sidecar container which is a HTTP proxy for the controller manager, +# it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.m.daocloud.io/kubebuilder/kube-rbac-proxy:v0.4.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--metrics-addr=127.0.0.1:8080" + - "--enable-leader-election" + - "--manager-pod=$(MY_POD_NAME)" + - "--manager-namespace=$(MY_POD_NAMESPACE)" diff --git a/config/default/manager_prometheus_metrics_patch.yaml b/config/default/manager_prometheus_metrics_patch.yaml new file mode 100644 index 0000000..0b96c68 --- /dev/null +++ b/config/default/manager_prometheus_metrics_patch.yaml @@ -0,0 +1,19 @@ +# This patch enables Prometheus scraping for the manager pod. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + metadata: + annotations: + prometheus.io/scrape: 'true' + spec: + containers: + # Expose the prometheus metrics on default port + - name: manager + ports: + - containerPort: 8080 + name: metrics + protocol: TCP diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..738de35 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 0000000..ac5693d --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..08e8194 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: daocloud.io/daocloud/kubeadm-operator + newTag: v0.0.2 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..6a8b0c5 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - command: + - /manager + args: + - "--mode=manager" + - "--manager-pod=$(MY_POD_NAME)" + - "--manager-namespace=$(MY_POD_NAMESPACE)" + - "--enable-leader-election" + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: daocloud.io/daocloud/kubeadm-operator:v0.0.2 + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..618f5e4 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..48ed1e4 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 0000000..d61e546 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "8443" + prometheus.io/scheme: https + prometheus.io/scrape: "true" + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..817f1fe --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 3 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..eaa7915 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,32 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..eed1690 --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..112039c --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - operations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - operations/status + verbs: + - get + - patch + - update +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - runtimetaskgroups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - runtimetaskgroups/status + verbs: + - get + - patch + - update +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - runtimetasks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.kubeadm.x-k8s.io + resources: + - runtimetasks/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..8f26587 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/samples/operator_v1alpha1_operation.yaml b/config/samples/operator_v1alpha1_operation.yaml new file mode 100644 index 0000000..71e008c --- /dev/null +++ b/config/samples/operator_v1alpha1_operation.yaml @@ -0,0 +1,8 @@ +apiVersion: operator.kubeadm.x-k8s.io/v1alpha1 +kind: Operation +metadata: + name: upgrade +spec: + executionMode: DryRun + upgrade: + kubernetesVersion: v1.23.4 diff --git a/config/samples/operator_v1alpha1_runtimetask.yaml b/config/samples/operator_v1alpha1_runtimetask.yaml new file mode 100644 index 0000000..e06bb11 --- /dev/null +++ b/config/samples/operator_v1alpha1_runtimetask.yaml @@ -0,0 +1,12 @@ +apiVersion: operator.kubeadm.x-k8s.io/v1alpha1 +kind: RuntimeTask +metadata: + name: task-sample +spec: + # Add fields here + nodeName: kind-control-plane + commands: + - pass: {} + - fail: {} + - wait: + seconds: 10 diff --git a/config/samples/operator_v1alpha1_runtimetaskgroup.yaml b/config/samples/operator_v1alpha1_runtimetaskgroup.yaml new file mode 100644 index 0000000..481c3d2 --- /dev/null +++ b/config/samples/operator_v1alpha1_runtimetaskgroup.yaml @@ -0,0 +1,21 @@ +apiVersion: operator.kubeadm.x-k8s.io/v1alpha1 +kind: RuntimeTaskGroup +metadata: + name: taskgroup-sample2 +spec: + nodeSelector: + matchLabels: + node-role.kubernetes.io/master: "" + selector: + matchLabels: + app: a + template: + metadata: + labels: + app: a + spec: + commands: + - pass: {} + - pass: {} + - wait: + seconds: 10 diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..25e21e3 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..31e0f82 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/operation_controller.go b/controllers/operation_controller.go new file mode 100644 index 0000000..23b1f44 --- /dev/null +++ b/controllers/operation_controller.go @@ -0,0 +1,366 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" + operatorerrors "k8s.io/kubeadm/operator/errors" + "k8s.io/kubeadm/operator/operations" +) + +// OperationReconciler reconciles a Operation object +type OperationReconciler struct { + client.Client + ManagerContainerName string + ManagerNamespace string + AgentImage string + MetricsRBAC bool + Log logr.Logger + recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch; +// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=operations,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=operations/status,verbs=get;update;patch + +// SetupWithManager configures the controller for calling the reconciler +func (r *OperationReconciler) SetupWithManager(mgr ctrl.Manager) error { + err := ctrl.NewControllerManagedBy(mgr). + For(&operatorv1.Operation{}). + Owns(&operatorv1.RuntimeTaskGroup{}). // force reconcile operation every time one of the owned TaskGroups change + Complete(r) + + //TODO: watch DS for operation Daemonsets + + r.recorder = mgr.GetEventRecorderFor("operation-controller") + return err +} + +// Reconcile an operation +func (r *OperationReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr error) { + ctx := context.Background() + log := r.Log.WithValues("operation", req.NamespacedName) + + // Fetch the Operation instance + operation := &operatorv1.Operation{} + if err := r.Client.Get(ctx, req.NamespacedName, operation); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Ignore the Operation if it is already completed or failed + if operation.Status.CompletionTime != nil { + // Reconcile the daemon set that deploys controller agents on nodes, so we are sure it is deleted after completion + err := r.reconcileDaemonSet(operation, log) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + // Initialize the patch helper + + patchHelper, err := patch.NewHelper(operation, r) + if err != nil { + return ctrl.Result{}, err + } + // Always attempt to Patch the Operation object and status after each reconciliation. + defer func() { + if err := patchHelper.Patch(ctx, operation); err != nil { + log.Error(err, "failed to patch Operation") + if rerr == nil { + rerr = err + } + } + }() + + // Reconcile the Operation + if err := r.reconcileOperation(operation, log); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *OperationReconciler) reconcileDaemonSet(operation *operatorv1.Operation, log logr.Logger) error { + daemonSet, err := getDaemonSet(r.Client, operation, r.ManagerNamespace) + if err != nil { + return err + } + + if daemonSet != nil { + // if operation completed + if daemonSetShouldBeRunning(operation) { + return nil + } + + log.WithValues("daemonset-name", daemonSetName(operation.Name)).Info("deleting DaemonSet") + if err := deleteDaemonSet(r.Client, daemonSet); err != nil { + return err + } + + return nil + } + + if !daemonSetShouldBeRunning(operation) { + return nil + } + + // if operation running + log.WithValues("daemonset-name", daemonSetName(operation.Name)).Info("creating DaemonSet") + image := r.AgentImage + if image == "" { + image, err = getImage(r.Client, r.ManagerNamespace, r.ManagerContainerName) + if err != nil { + return err + } + } + + if err := createDaemonSet(r.Client, operation, r.ManagerNamespace, image, r.MetricsRBAC); err != nil { + return err + } + + return nil +} + +func daemonSetShouldBeRunning(operation *operatorv1.Operation) bool { + return operation.Status.CompletionTime == nil && + operation.Status.ErrorReason == nil && + operation.Status.ErrorMessage == nil +} + +func (r *OperationReconciler) reconcileOperation(operation *operatorv1.Operation, log logr.Logger) (err error) { + // Reconcile paused settings + r.reconcilePause(operation) + + // Reconcile labels so the operation and the operation object can be searched by a well-known set of labels + r.reconcileLabels(operation) + + // Reconcile the daemon set that deploys controller agents on nodes + err = r.reconcileDaemonSet(operation, log) + if err != nil { + return + } + + // Handle deleted Operation + if !operation.DeletionTimestamp.IsZero() { + err = r.reconcileDelete(operation) + if err != nil { + return + } + } + // Handle non-deleted Operation + + // gets controlled taskGroups items (desired vs actual) + taskGroups, err := r.reconcileTaskGroups(operation, log) + if err != nil { + return err + } + + err = r.reconcileNormal(operation, taskGroups, log) + if err != nil { + return + } + + // Always reconcile Phase at the end + r.reconcilePhase(operation) + + return +} + +func (r *OperationReconciler) reconcilePause(operation *operatorv1.Operation) { + // record paused state change, if any + recordPausedChange(r.recorder, operation, operation.Status.Paused, operation.Spec.Paused) + + // update status with paused setting + operation.Status.Paused = operation.Spec.Paused +} + +func (r *OperationReconciler) reconcileLabels(operation *operatorv1.Operation) { + if operation.Labels == nil { + operation.Labels = map[string]string{} + } + if _, ok := operation.Labels[operatorv1.OperationNameLabel]; !ok { + operation.Labels[operatorv1.OperationNameLabel] = operation.Name + } + if _, ok := operation.Labels[operatorv1.OperationUIDLabel]; !ok { + operation.Labels[operatorv1.OperationUIDLabel] = string(uuid.NewUUID()) + } +} + +func (r *OperationReconciler) reconcileTaskGroups(operation *operatorv1.Operation, log logr.Logger) (*taskGroupReconcileList, error) { + // gets all the desired TaskGroup objects for the current operation + // Nb. this is the domain knowledge encoded into operation implementations + desired, err := operations.TaskGroupList(operation) + if err != nil { + return nil, errors.Wrap(err, "failed to get desired TaskGroup list") + } + + // gets the current TaskGroup objects related to this Operation + actual, err := listTaskGroupsByLabels(r.Client, operation.Labels) + if err != nil { + return nil, errors.Wrap(err, "failed to list TaskGroup") + } + + r.Log.Info("reconciling", "desired-TaskGroups", len(desired.Items), "TaskGroups", len(actual.Items)) + + // match current and desired TaskGroup, so the controller can determine what is necessary to do next + taskGroups := reconcileTaskGroups(desired, actual) + + // update replica counters + operation.Status.Groups = int32(len(taskGroups.all)) + operation.Status.RunningGroups = int32(len(taskGroups.running)) + operation.Status.SucceededGroups = int32(len(taskGroups.completed)) + operation.Status.FailedGroups = int32(len(taskGroups.failed)) + operation.Status.InvalidGroups = int32(len(taskGroups.invalid)) + + return taskGroups, nil +} + +func (r *OperationReconciler) reconcileNormal(operation *operatorv1.Operation, taskGroups *taskGroupReconcileList, log logr.Logger) error { + // If the Operation doesn't have finalizer, add it. + //if !util.Contains(operation.Finalizers, operatorv1.OperationFinalizer) { + // operation.Finalizers = append(operation.Finalizers, operatorv1.OperationFinalizer) + //} + + // if there are TaskGroup not yet completed (pending or running), cleanup error messages (required e.g. after recovery) + // NB. It is necessary to give priority to running vs errors so the operation controller keeps alive/restarts + // the DaemonsSet for processing tasks + if taskGroups.activeTaskGroups() > 0 { + operation.Status.ResetError() + } else { + // if there are invalid combinations (e.g. a TaskGroup without a desired TaskGroup) + // set the error and stop creating new TaskGroups + if len(taskGroups.invalid) > 0 { + // TODO: improve error message + operation.Status.SetError( + operatorerrors.NewOperationReconciliationError("something invalid"), + ) + return nil + } + + // if there are failed TaskGroup + // set the error and stop creating new TaskGroups + if len(taskGroups.failed) > 0 { + // TODO: improve error message + operation.Status.SetError( + operatorerrors.NewOperationReplicaError("something failed"), + ) + return nil + } + } + + // TODO: manage adopt tasks/tasks to be orphaned + + // if nil, set the Operation start time + if operation.Status.StartTime == nil { + operation.Status.SetStartTime() + + //TODO: add a signature so we can detect if someone/something changes the operations while it is processed + return nil + } + + // if the completed TaskGroup have reached the number of expected TaskGroup, the Operation is completed + // NB. we are doing this before checking pause because if everything is completed, does not make sense to pause + if len(taskGroups.completed) == len(taskGroups.all) { + // NB. we are setting this condition explicitly in order to avoid that the Operation accidentally + // restarts to create TaskGroup + operation.Status.SetCompletionTime() + } + + // if the TaskGroup is paused, return + if operation.Status.Paused { + return nil + } + + // otherwise, proceed creating TaskGroup + + // if there are still TaskGroup to be created + if len(taskGroups.tobeCreated) > 0 { + // if there no TaskGroup not yet completed (pending or running) + if taskGroups.activeTaskGroups() == 0 { + // create the next TaskGroup in the ordered sequence + nextTaskGroup := taskGroups.tobeCreated[0].planned + log.WithValues("task-group", nextTaskGroup.Name).Info("creating task") + + err := r.Client.Create(context.Background(), nextTaskGroup) + if err != nil { + return errors.Wrap(err, "Failed to create TaskGroup") + } + } + } + + return nil +} + +func (r *OperationReconciler) reconcileDelete(operation *operatorv1.Operation) error { + + // Operation is deleted so remove the finalizer. + //operation.Finalizers = util.Filter(operation.Finalizers, operatorv1.OperationFinalizer) + + return nil +} + +func (r *OperationReconciler) reconcilePhase(operation *operatorv1.Operation) { + // Set the phase to "deleting" if the deletion timestamp is set. + if !operation.DeletionTimestamp.IsZero() { + operation.Status.SetTypedPhase(operatorv1.OperationPhaseDeleted) + return + } + + // Set the phase to "failed" if any of Status.ErrorReason or Status.ErrorMessage is not-nil. + if operation.Status.ErrorReason != nil || operation.Status.ErrorMessage != nil { + operation.Status.SetTypedPhase(operatorv1.OperationPhaseFailed) + return + } + + // Set the phase to "succeeded" if completion date is set. + if operation.Status.CompletionTime != nil { + operation.Status.SetTypedPhase(operatorv1.OperationPhaseSucceeded) + return + } + + // Set the phase to "paused" if paused set. + if operation.Status.Paused { + operation.Status.SetTypedPhase(operatorv1.OperationPhasePaused) + return + } + + // Set the phase to "running" if start date is set. + if operation.Status.StartTime != nil { + operation.Status.SetTypedPhase(operatorv1.OperationPhaseRunning) + return + } + + // Set the phase to "pending" if nil. + operation.Status.SetTypedPhase(operatorv1.OperationPhasePending) +} diff --git a/controllers/operation_controller_test.go b/controllers/operation_controller_test.go new file mode 100644 index 0000000..0c4f50e --- /dev/null +++ b/controllers/operation_controller_test.go @@ -0,0 +1,634 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func TestOperatorReconcilePhase(t *testing.T) { + tx := metav1.Now() + errMessage := "error" + + tests := []struct { + name string + input *operatorv1.Operation + expected *operatorv1.Operation + }{ + { + name: "Reconcile pending state", + input: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{}, + }, + expected: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + Phase: string(operatorv1.OperationPhasePending), + }, + }, + }, + { + name: "Reconcile running state", + input: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + }, + }, + expected: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + Phase: string(operatorv1.RuntimeTaskPhaseRunning), + }, + }, + }, + { + name: "Reconcile paused state", + input: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + Paused: true, + }, + }, + expected: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + Paused: true, + Phase: string(operatorv1.OperationPhasePaused), + }, + }, + }, + { + name: "Reconcile succeeded state", + input: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + CompletionTime: &tx, + }, + }, + expected: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + CompletionTime: &tx, + Phase: string(operatorv1.OperationPhaseSucceeded), + }, + }, + }, + { + name: "Reconcile failed state", + input: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + ErrorMessage: &errMessage, + }, + }, + expected: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: &tx, + ErrorMessage: &errMessage, + Phase: string(operatorv1.OperationPhaseFailed), + }, + }, + }, + { + name: "Reconcile deleted state", + input: &operatorv1.Operation{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &tx, + }, + }, + expected: &operatorv1.Operation{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &tx, + }, + Status: operatorv1.OperationStatus{ + Phase: string(operatorv1.OperationPhaseDeleted), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &OperationReconciler{} + r.reconcilePhase(tt.input) + + if !reflect.DeepEqual(tt.input, tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, tt.input) + } + }) + } +} + +func TestOperatorReconcilePause(t *testing.T) { + type expected struct { + operation *operatorv1.Operation + events int + } + + tests := []struct { + name string + input *operatorv1.Operation + expected expected + }{ + { + name: "Reconcile an Operation not paused with Spec paused", + input: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: true, + }, + Status: operatorv1.OperationStatus{ + Paused: false, + }, + }, + expected: expected{ + operation: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: true, + }, + Status: operatorv1.OperationStatus{ + Paused: true, + }, + }, + events: 1, + }, + }, + { + name: "Reconcile an Operation paused with Spec not paused", + input: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: false, + }, + Status: operatorv1.OperationStatus{ + Paused: true, + }, + }, + expected: expected{ + operation: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: false, + }, + Status: operatorv1.OperationStatus{ + Paused: false, + }, + }, + events: 1, + }, + }, + { + name: "Reconcile an Operation paused with Spec paused", + input: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: true, + }, + Status: operatorv1.OperationStatus{ + Paused: true, + }, + }, + expected: expected{ + operation: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: true, + }, + Status: operatorv1.OperationStatus{ + Paused: true, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile an Operation not paused with Spec not paused", + input: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: false, + }, + Status: operatorv1.OperationStatus{ + Paused: false, + }, + }, + expected: expected{ + operation: &operatorv1.Operation{ + Spec: operatorv1.OperationSpec{ + Paused: false, + }, + Status: operatorv1.OperationStatus{ + Paused: false, + }, + }, + events: 0, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(1) + + r := &OperationReconciler{ + recorder: rec, + } + + r.reconcilePause(tt.input) + + if !reflect.DeepEqual(tt.input, tt.expected.operation) { + t.Errorf("expected %v, got %v", tt.expected.operation, tt.input) + } + + if tt.expected.events != len(rec.Events) { + t.Errorf("expected %v, got %v", tt.expected.events, len(rec.Events)) + } + }) + } +} + +func TestOperationReconciler_Reconcile(t *testing.T) { + type fields struct { + Objs []runtime.Object + } + type args struct { + req ctrl.Request + } + tests := []struct { + name string + fields fields + args args + want ctrl.Result + wantErr bool + }{ + { + name: "Reconcile does nothing if operation does not exist", + fields: fields{}, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile does nothing if the operation is already completed", + fields: fields{ + Objs: []runtime.Object{ + &operatorv1.Operation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Operation", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-operation", + }, + Status: operatorv1.OperationStatus{ + CompletionTime: timePtr(metav1.Now()), + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-operation"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile pass", + fields: fields{ + Objs: []runtime.Object{ + &operatorv1.Operation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Operation", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-operation", + }, + Spec: operatorv1.OperationSpec{ + OperatorDescriptor: operatorv1.OperatorDescriptor{ + CustomOperation: &operatorv1.CustomOperationSpec{}, + }, + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-operation"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &OperationReconciler{ + Client: fake.NewFakeClientWithScheme(setupScheme(), tt.fields.Objs...), + AgentImage: "some-image", //making reconcile operation pass + MetricsRBAC: false, + Log: log.Log, + recorder: record.NewFakeRecorder(1), + } + got, err := r.Reconcile(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("Reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Reconcile() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestOperationReconciler_reconcileNormal(t *testing.T) { + type args struct { + operation *operatorv1.Operation + taskGroups *taskGroupReconcileList + } + type want struct { + operation *operatorv1.Operation + } + tests := []struct { + name string + args args + want want + wantTaskGroups int + wantErr bool + }{ + { + name: "Reconcile sets error if a taskGroup is failed and no taskGroup is active", + args: args{ + operation: &operatorv1.Operation{}, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + }, + failed: []*taskGroupReconcileItem{ + {}, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + ErrorMessage: stringPtr("error"), //using error as a marker for "whatever error was raised" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile sets error if a taskGroup is invalid and no taskGroup is active", + args: args{ + operation: &operatorv1.Operation{}, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + }, + invalid: []*taskGroupReconcileItem{ + {}, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + ErrorMessage: stringPtr("error"), //using error as a marker for "whatever error was raised" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile set start time", + args: args{ + operation: &operatorv1.Operation{}, + taskGroups: &taskGroupReconcileList{}, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile reset error if a taskGroup is active", + args: args{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + ErrorMessage: stringPtr("error"), + }, + }, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + {}, + }, + running: []*taskGroupReconcileItem{ + {}, + }, + failed: []*taskGroupReconcileItem{ // failed should be ignored if a taskGroup is active + {}, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile set completion time if no more taskGroup to create", + args: args{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + taskGroups: &taskGroupReconcileList{}, //empty list of nodes -> no more task to create + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CompletionTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it completed" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile do nothing if paused", + args: args{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Now()), + Paused: true, + }, + }, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + }, + tobeCreated: []*taskGroupReconcileItem{ + {}, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + Paused: true, + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + { + name: "Reconcile creates a taskGroup if nothing running", + args: args{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + }, + tobeCreated: []*taskGroupReconcileItem{ + { + planned: &operatorv1.RuntimeTaskGroup{}, + }, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTaskGroups: 1, + wantErr: false, + }, + { + name: "Reconcile does not creates a taskGroup if something running", + args: args{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + taskGroups: &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{ + {}, + {}, + }, + tobeCreated: []*taskGroupReconcileItem{ + {}, + }, + running: []*taskGroupReconcileItem{ + {}, + }, + }, + }, + want: want{ + operation: &operatorv1.Operation{ + Status: operatorv1.OperationStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTaskGroups: 0, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := fake.NewFakeClientWithScheme(setupScheme()) + + r := &OperationReconciler{ + Client: c, + recorder: record.NewFakeRecorder(1), + Log: log.Log, + } + if err := r.reconcileNormal(tt.args.operation, tt.args.taskGroups, r.Log); (err != nil) != tt.wantErr { + t.Errorf("reconcileNormal() error = %v, wantErr %v", err, tt.wantErr) + } + + fixupWantOperation(tt.want.operation, tt.args.operation) + + if !reflect.DeepEqual(tt.args.operation, tt.want.operation) { + t.Errorf("reconcileNormal() = %v, want %v", tt.args.operation, tt.want.operation) + } + + taskGroups := &operatorv1.RuntimeTaskGroupList{} + if err := c.List(context.Background(), taskGroups); err != nil { + t.Fatalf("List() error = %v", err) + } + + if len(taskGroups.Items) != tt.wantTaskGroups { + t.Errorf("reconcileNormal() = %v taskGroups, want %v taskGroups", len(taskGroups.Items), tt.wantTaskGroups) + } + }) + } +} + +func fixupWantOperation(want *operatorv1.Operation, got *operatorv1.Operation) { + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.CreationTimestamp.IsZero() { + want.CreationTimestamp = got.CreationTimestamp + } + + // In case want.ErrorMessage is a marker, replace it with the current error + if want.Status.ErrorMessage != nil && *want.Status.ErrorMessage == "error" && got.Status.ErrorMessage != nil { + want.Status.ErrorMessage = got.Status.ErrorMessage + want.Status.ErrorReason = got.Status.ErrorReason + } + + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.Status.StartTime != nil && want.Status.StartTime.IsZero() && got.Status.StartTime != nil { + want.Status.StartTime = got.Status.StartTime + } + // In case want.CompletionTime is a marker, replace it with the current CompletionTime + if want.Status.CompletionTime != nil && want.Status.CompletionTime.IsZero() && got.Status.CompletionTime != nil { + want.Status.CompletionTime = got.Status.CompletionTime + } +} diff --git a/controllers/runtimetask_controller.go b/controllers/runtimetask_controller.go new file mode 100644 index 0000000..0630d4a --- /dev/null +++ b/controllers/runtimetask_controller.go @@ -0,0 +1,328 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" + commandimpl "k8s.io/kubeadm/operator/commands" + operatorerrors "k8s.io/kubeadm/operator/errors" +) + +// RuntimeTaskReconciler reconciles a RuntimeTask object +type RuntimeTaskReconciler struct { + client.Client + NodeName string + Operation string + recorder record.EventRecorder + Log logr.Logger +} + +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=runtimetasks,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=runtimetasks/status,verbs=get;update;patch + +// SetupWithManager configures the controller for calling the reconciler +func (r *RuntimeTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { + var mapFunc handler.ToRequestsFunc = func(o handler.MapObject) []reconcile.Request { + return taskGroupToTaskRequests(r.Client, o) + } + + err := ctrl.NewControllerManagedBy(mgr). + For(&operatorv1.RuntimeTask{}). + Watches( // force reconcile Task every time the parent TaskGroup changes + &source.Kind{Type: &operatorv1.RuntimeTaskGroup{}}, + &handler.EnqueueRequestsFromMapFunc{ToRequests: mapFunc}, + ). + Complete(r) + + r.recorder = mgr.GetEventRecorderFor("runtime-task-controller") + return err +} + +// Reconcile a runtimetask +func (r *RuntimeTaskReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr error) { + ctx := context.Background() + log := r.Log.WithValues("task", req.NamespacedName) + + // Fetch the Task instance + task := &operatorv1.RuntimeTask{} + if err := r.Client.Get(ctx, req.NamespacedName, task); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Ignore the Task if it doesn't target the node the controller is supervising + if task.Spec.NodeName != r.NodeName { + return ctrl.Result{}, nil + } + + // Ignore the Task if it is already completed or failed + if task.Status.CompletionTime != nil { + return ctrl.Result{}, nil + } + + // Fetch the parent TaskGroup instance + taskgroup, err := getOwnerTaskGroup(ctx, r.Client, task.ObjectMeta) + if err != nil { + return ctrl.Result{}, err + } + + // Fetch the parent Operation instance + operation, err := getOwnerOperation(ctx, r.Client, taskgroup.ObjectMeta) + if err != nil { + return ctrl.Result{}, err + } + + // If the controller is set to manage Task for a specific operation, ignore everything else + if r.Operation != operation.Name { + return ctrl.Result{}, nil + } + + // Initialize the patch helper + patchHelper, err := patch.NewHelper(task, r) + if err != nil { + return ctrl.Result{}, err + } + // Always attempt to Patch the Task object and status after each reconciliation. + defer func() { + if err := patchHelper.Patch(ctx, task); err != nil { + log.Error(err, "failed to patch Task") + if rerr == nil { + rerr = err + } + } + }() + + // Reconcile the Task + if err := r.reconcileTask(operation, taskgroup, task, log); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +func (r *RuntimeTaskReconciler) reconcileTask(operation *operatorv1.Operation, taskgroup *operatorv1.RuntimeTaskGroup, task *operatorv1.RuntimeTask, log logr.Logger) (err error) { + // gets relevant settings from top level objects + executionMode := operation.Spec.GetTypedOperationExecutionMode() + operationPaused := operation.Status.Paused + + // Reconcile recovery from errors + recovered := r.reconcileRecovery(executionMode, task, log) + + // Reconcile paused override from top level objects + r.reconcilePauseOverride(operationPaused, task) + + // Handle deleted Task + if !task.DeletionTimestamp.IsZero() { + err = r.reconcileDelete(task) + if err != nil { + return + } + } + // Handle non-deleted/non-recovered Task + // NB. in case of a task recovered from error, we are forcing another reconcile before actually + // executing the next command so the user get evidence of what is happening + if !recovered { + err = r.reconcileNormal(executionMode, task, log) + if err != nil { + return + } + } + + // Always reconcile Task Phase at the end + r.reconcilePhase(task) + + return +} + +func (r *RuntimeTaskReconciler) reconcileRecovery(executionMode operatorv1.OperationExecutionMode, task *operatorv1.RuntimeTask, log logr.Logger) bool { + // if there is no error, return + if task.Status.ErrorReason == nil && task.Status.ErrorMessage == nil { + return false + } + + // if there is no error recovery strategy, return + if task.Spec.GetTypedTaskRecoveryStrategy() == operatorv1.RuntimeTaskRecoveryUnknownStrategy { + return false + } + + switch task.Spec.GetTypedTaskRecoveryStrategy() { + case operatorv1.RuntimeTaskRecoveryRetryingFailedCommandStrategy: + log.WithValues("command", task.Status.CurrentCommand).Info("Retrying command after failure") + r.recorder.Event(task, corev1.EventTypeNormal, "TaskErrorRetry", fmt.Sprintf("Retrying command %d after failure", task.Status.CurrentCommand)) + case operatorv1.RuntimeTaskRecoverySkippingFailedCommandStrategy: + log.WithValues("command", task.Status.CurrentCommand).Info("Skipping command after failure") + r.recorder.Event(task, corev1.EventTypeNormal, "TaskErrorSkip", fmt.Sprintf("Skipping command %d after failure", task.Status.CurrentCommand)) + + // if all the commands are done, set the Task completion time + if int(task.Status.CurrentCommand) >= len(task.Spec.Commands) { + task.Status.SetCompletionTime() + } else { + // Move to the next command + task.Status.NextCurrentCommand(task.Spec.Commands) + if executionMode == operatorv1.OperationExecutionModeControlled { + task.Status.Paused = true + } + } + + default: + //TODO: error (if possible do validation before getting here) + } + + // Reset the error + task.Status.ResetError() + + // Reset the recovery mode so the user can choose again how to proceed at the next error + task.Spec.RecoveryMode = "" + + return true +} + +func (r *RuntimeTaskReconciler) reconcilePauseOverride(operationPaused bool, task *operatorv1.RuntimeTask) { + // record paused override state change, if any + pausedOverride := operationPaused + recordPausedChange(r.recorder, task, task.Status.Paused, pausedOverride, "by top level objects") + + // update status with paused override setting from top level objects + task.Status.Paused = pausedOverride +} + +func (r *RuntimeTaskReconciler) reconcileNormal(executionMode operatorv1.OperationExecutionMode, task *operatorv1.RuntimeTask, log logr.Logger) error { + // If the Task doesn't have finalizer, add it. + //if !util.Contains(task.Finalizers, operatorv1.RuntimeTaskFinalizer) { + // task.Finalizers = append(task.Finalizers, operatorv1.RuntimeTaskFinalizer) + //} + + // if higher level object are paused, return + if task.Status.Paused { + return nil + } + + // if nil, set the Task start time, initialize CurrentCommand and return + // NB. we are returning here so the object get updated reporting start condition + // before actual execution starts + if task.Status.StartTime == nil { + task.Status.SetStartTime() + task.Status.NextCurrentCommand(task.Spec.Commands) + return nil + } + + // Proceed with the current command execution + + if executionMode == operatorv1.OperationExecutionModeDryRun { + // if dry running wait for an arbitrary delay so the user will get a better perception of the Task execution order + time.Sleep(3 * time.Second) + } else { + // else we should execute the CurrentCommand + log.WithValues("command", task.Status.CurrentCommand).Info("running command") + + // transpose CurrentCommand (1 based) to index (0 based) and check index out of range + index := int(task.Status.CurrentCommand) - 1 + if index < 0 || index >= len(task.Spec.Commands) { + task.Status.SetError( + operatorerrors.NewRuntimeTaskIndexOutOfRangeError("command with index %d does not exists for task %s", index, task.Name), + ) + } + + // execute the command + err := commandimpl.RunCommand(&task.Spec.Commands[index], log) + + // if the command returned an error, return + if err != nil { + log.WithValues("command", task.Status.CurrentCommand).WithValues("error", fmt.Sprintf("%+v", err)).Info("command failed") + r.recorder.Event(task, corev1.EventTypeWarning, "CommandError", fmt.Sprintf("Command %d execution failed: %s", task.Status.CurrentCommand, fmt.Sprintf("%+v", err))) + task.Status.SetError( + operatorerrors.NewRuntimeTaskExecutionError("error executing command number %d for task %s: %+v", task.Status.CurrentCommand, task.Name, err), + ) + return nil + } + + log.WithValues("command", task.Status.CurrentCommand).Info("command completed") + r.recorder.Event(task, corev1.EventTypeNormal, "CommandCompleted", fmt.Sprintf("Command %d execution completed", task.Status.CurrentCommand)) + } + + // if all the commands are done, set the Task completion time and return + if int(task.Status.CurrentCommand) >= len(task.Spec.Commands) { + task.Status.SetCompletionTime() + return nil + } + + // Otherwise, move to the next command + task.Status.NextCurrentCommand(task.Spec.Commands) + if executionMode == operatorv1.OperationExecutionModeControlled { + task.Status.Paused = true + } + return nil +} + +func (r *RuntimeTaskReconciler) reconcileDelete(task *operatorv1.RuntimeTask) error { + + // Task is deleted so remove the finalizer. + //task.Finalizers = util.Filter(task.Finalizers, operatorv1.RuntimeTaskFinalizer) + + return nil +} + +func (r *RuntimeTaskReconciler) reconcilePhase(task *operatorv1.RuntimeTask) { + // Set the phase to "deleting" if the deletion timestamp is set. + if !task.DeletionTimestamp.IsZero() { + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhaseDeleted) + return + } + + // Set the phase to "failed" if any of Status.ErrorReason or Status.ErrorMessage is not-nil. + if task.Status.ErrorReason != nil || task.Status.ErrorMessage != nil { + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhaseFailed) + return + } + + // Set the phase to "succeeded" if completion date is set. + if task.Status.CompletionTime != nil { + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhaseSucceeded) + return + } + + // Set the phase to "paused" if paused is set. + if task.Status.Paused { + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhasePaused) + return + } + + // Set the phase to "running" if start date is set. + if task.Status.StartTime != nil { + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhaseRunning) + return + } + + // Set the phase to "pending" if nil. + task.Status.SetTypedPhase(operatorv1.RuntimeTaskPhasePending) +} diff --git a/controllers/runtimetask_controller_test.go b/controllers/runtimetask_controller_test.go new file mode 100644 index 0000000..642e8aa --- /dev/null +++ b/controllers/runtimetask_controller_test.go @@ -0,0 +1,1138 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + "k8s.io/kubeadm/operator/errors" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func TestRuntimeTaskReconciler_reconcilePhase(t *testing.T) { + tx := timePtr(metav1.Now()) + + type args struct { + task *operatorv1.RuntimeTask + } + type want struct { + task *operatorv1.RuntimeTask + } + tests := []struct { + name string + args args + want want + }{ + { + name: "Reconcile pending state", + args: args{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{}, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Phase: string(operatorv1.RuntimeTaskPhasePending), + }, + }, + }, + }, + { + name: "Reconcile running state", + args: args{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + Phase: string(operatorv1.RuntimeTaskPhaseRunning), + }, + }, + }, + }, + { + name: "Reconcile paused state", + args: args{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + Paused: true, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + Paused: true, + Phase: string(operatorv1.RuntimeTaskPhasePaused), + }, + }, + }, + }, + { + name: "Reconcile succeeded state", + args: args{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + CompletionTime: tx, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + CompletionTime: tx, + Phase: string(operatorv1.RuntimeTaskPhaseSucceeded), + }, + }, + }, + }, + { + name: "Reconcile failed state", + args: args{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + ErrorMessage: stringPtr("error"), + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + StartTime: tx, + ErrorMessage: stringPtr("error"), + Phase: string(operatorv1.RuntimeTaskPhaseFailed), + }, + }, + }, + }, + { + name: "Reconcile deleted state", + args: args{ + task: &operatorv1.RuntimeTask{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: tx, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: tx, + }, + Status: operatorv1.RuntimeTaskStatus{ + Phase: string(operatorv1.RuntimeTaskPhaseDeleted), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RuntimeTaskReconciler{} + r.reconcilePhase(tt.args.task) + + if !reflect.DeepEqual(tt.args.task, tt.want.task) { + t.Errorf("reconcilePhase() = %v, want %v", tt.args.task, tt.want.task) + } + }) + } +} + +func TestRuntimeTaskReconciler_reconcilePauseOverride(t *testing.T) { + type args struct { + operationPaused bool + task *operatorv1.RuntimeTask + } + type want struct { + task *operatorv1.RuntimeTask + events int + } + tests := []struct { + name string + args args + want want + }{ + { + name: "Reconcile a Task not paused with an Operation not paused is NOP", + args: args{ + operationPaused: false, + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: false, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: false, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a Task not paused with an Operation currently paused set pause", + args: args{ + operationPaused: true, + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: false, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + events: 1, + }, + }, + { + name: "Reconcile a Task paused with an Operation currently paused is NOP", + args: args{ + operationPaused: true, + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a Task paused with an Operation currently not paused unset pause", + args: args{ + operationPaused: false, + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: false, + }, + }, + events: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(1) + + r := &RuntimeTaskReconciler{ + Client: nil, + NodeName: "", + Operation: "", + recorder: rec, + Log: nil, + } + + r.reconcilePauseOverride(tt.args.operationPaused, tt.args.task) + + if !reflect.DeepEqual(tt.args.task, tt.want.task) { + t.Errorf("reconcilePauseOverride() = %v, want %v", tt.args.task, tt.want.task) + } + + if tt.want.events != len(rec.Events) { + t.Errorf("reconcilePauseOverride() = %v events recorded, want %v events", tt.want.events, len(rec.Events)) + } + }) + } +} + +func TestRuntimeTaskReconciler_reconcileRecovery(t *testing.T) { + type args struct { + executionMode operatorv1.OperationExecutionMode + task *operatorv1.RuntimeTask + } + type want struct { + ret bool + task *operatorv1.RuntimeTask + events int + } + tests := []struct { + name string + args args + want want + }{ + { + name: "Reconcile a Task without errors is NOP", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + }, + }, + want: want{ + ret: false, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a Task without recovery strategy is NOP", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + }, + }, + }, + want: want{ + ret: false, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a Task using RetryError strategy removes the error (and keep CurrentCommand)", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: string(operatorv1.RuntimeTaskRecoveryRetryingFailedCommandStrategy), + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + CurrentCommand: 1, + }, + }, + }, + want: want{ + ret: true, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: "", // recovery strategy back to empty + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: nil, // error removed + ErrorMessage: nil, + CurrentCommand: 1, + }, + }, + events: 1, + }, + }, + { + name: "Reconcile a Task using SkipError strategy removes the error and moves to the next command", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: string(operatorv1.RuntimeTaskRecoverySkippingFailedCommandStrategy), + Commands: []operatorv1.CommandDescriptor{ + {}, + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + CurrentCommand: 1, // same command + }, + }, + }, + want: want{ + ret: true, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: "", // recovery strategy back to empty + Commands: []operatorv1.CommandDescriptor{ + {}, + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: nil, // error removed + ErrorMessage: nil, + CurrentCommand: 2, // next command + CommandProgress: "2/2", + }, + }, + events: 1, + }, + }, + { + name: "Reconcile a Task using SkipError strategy removes the error and moves to the next command + set pause if Mode=Controlled", + args: args{ + executionMode: operatorv1.OperationExecutionModeControlled, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: string(operatorv1.RuntimeTaskRecoverySkippingFailedCommandStrategy), + Commands: []operatorv1.CommandDescriptor{ + {}, + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + CurrentCommand: 1, + }, + }, + }, + want: want{ + ret: true, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: "", // recovery strategy back to empty + Commands: []operatorv1.CommandDescriptor{ + {}, + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: nil, // error removed + ErrorMessage: nil, + CurrentCommand: 2, // next command + CommandProgress: "2/2", + Paused: true, // paused + }, + }, + events: 1, + }, + }, + { + name: "Reconcile a Task using SkipError strategy removes the error and set completed if there are no more commands", + args: args{ + executionMode: operatorv1.OperationExecutionModeControlled, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: string(operatorv1.RuntimeTaskRecoverySkippingFailedCommandStrategy), + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + ErrorReason: runtimeTaskStatusErrorPtr(errors.RuntimeTaskExecutionError), + ErrorMessage: stringPtr("error"), + CurrentCommand: 1, + }, + }, + }, + want: want{ + ret: true, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + RecoveryMode: "", // recovery strategy back to empty + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + CompletionTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it completes" + ErrorReason: nil, // error removed + ErrorMessage: nil, + CurrentCommand: 1, // next command + }, + }, + events: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(1) + + r := &RuntimeTaskReconciler{ + Client: nil, + NodeName: "", + Operation: "", + recorder: rec, + Log: log.Log, + } + + if got := r.reconcileRecovery(tt.args.executionMode, tt.args.task, log.Log); got != tt.want.ret { + t.Errorf("reconcileRecovery() = %v, want %v", got, tt.want.ret) + } + + fixupWantTask(tt.want.task, tt.args.task) + + if !reflect.DeepEqual(tt.args.task, tt.want.task) { + t.Errorf("reconcileRecovery() = %v, want %v", tt.args.task, tt.want.task) + } + + if tt.want.events != len(rec.Events) { + t.Errorf("reconcileRecovery() = %v events recorded, want %v events", tt.want.events, len(rec.Events)) + } + }) + } +} + +func TestRuntimeTaskReconciler_reconcileNormal(t *testing.T) { + type args struct { + executionMode operatorv1.OperationExecutionMode + task *operatorv1.RuntimeTask + } + type want struct { + task *operatorv1.RuntimeTask + events int + } + tests := []struct { + name string + args args + want want + wantErr bool + }{ + { + name: "Reconcile paused task is a NOP", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Status: operatorv1.RuntimeTaskStatus{ + Paused: true, + }, + }, + events: 0, + }, + wantErr: false, + }, + { + name: "Reconcile new task sets start time", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CurrentCommand: 1, + CommandProgress: "1/1", + }, + }, + events: 0, + }, + wantErr: false, + }, + { + name: "Reconcile task already started run commands and move to the next command", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + CurrentCommand: 1, + CommandProgress: "1/2", + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CurrentCommand: 2, + CommandProgress: "2/2", + }, + }, + events: 1, + }, + wantErr: false, + }, + { + name: "Reconcile task already started run commands and move to the next command + pause if operation mode=controlled", + args: args{ + executionMode: operatorv1.OperationExecutionModeControlled, + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + CurrentCommand: 1, + CommandProgress: "1/2", + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CurrentCommand: 2, + CommandProgress: "2/2", + Paused: true, + }, + }, + events: 1, + }, + wantErr: false, + }, + { + name: "Reconcile task already started run commands and completes if no more commands", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + CurrentCommand: 1, + CommandProgress: "1/1", + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Pass: &operatorv1.PassCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CurrentCommand: 1, + CommandProgress: "1/1", + CompletionTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it completed" + }, + }, + events: 1, + }, + wantErr: false, + }, + { + name: "Reconcile task handle command failures", + args: args{ + executionMode: "", + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Fail: &operatorv1.FailCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + CurrentCommand: 1, + CommandProgress: "1/1", + }, + }, + }, + want: want{ + task: &operatorv1.RuntimeTask{ + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + { + Fail: &operatorv1.FailCommandSpec{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CurrentCommand: 1, + CommandProgress: "1/1", + ErrorMessage: stringPtr("error"), //using error as a marker for "whatever error there is" + }, + }, + events: 1, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(1) + + r := &RuntimeTaskReconciler{ + Client: nil, + NodeName: "", + Operation: "", + recorder: rec, + Log: log.Log, + } + if err := r.reconcileNormal(tt.args.executionMode, tt.args.task, log.Log); (err != nil) != tt.wantErr { + t.Errorf("reconcileNormal() error = %v, wantErr %v", err, tt.wantErr) + } + + fixupWantTask(tt.want.task, tt.args.task) + + if !reflect.DeepEqual(tt.args.task, tt.want.task) { + t.Errorf("reconcileRecovery() = %v, want %v", tt.args.task, tt.want.task) + } + + if tt.want.events != len(rec.Events) { + t.Errorf("reconcileRecovery() = %v events recorded, want %v events", tt.want.events, len(rec.Events)) + } + }) + } +} + +func TestRuntimeTaskReconciler_Reconcile(t *testing.T) { + type fields struct { + NodeName string + Operation string + Objs []runtime.Object + } + type args struct { + req ctrl.Request + } + tests := []struct { + name string + fields fields + args args + want ctrl.Result + wantErr bool + }{ + { + name: "Reconcile does nothing if task does not exist", + fields: fields{}, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile does nothing if task doesn't target the node the controller is supervising", + fields: fields{ + NodeName: "foo-node", + Objs: []runtime.Object{ + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "bar-task", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "bar-node", + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "bar-task"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile does nothing if the task is already completed", + fields: fields{ + NodeName: "foo-node", + Objs: []runtime.Object{ + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-task", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + }, + Status: operatorv1.RuntimeTaskStatus{ + CompletionTime: timePtr(metav1.Now()), + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-task"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile fails if failing to retrieve parent taskgroup", + fields: fields{ + NodeName: "foo-node", + Objs: []runtime.Object{ + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-task", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-task"}}, + }, + want: ctrl.Result{}, + wantErr: true, + }, + { + name: "Reconcile fails if failing to retrieve parent operation", + fields: fields{ + NodeName: "foo-node", + Objs: []runtime.Object{ + &operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTaskGroup", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup", + }, + }, + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-task", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "RuntimeTaskGroup", + Name: "foo-taskgroup", + }, + }, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-task"}}, + }, + want: ctrl.Result{}, + wantErr: true, + }, + { + name: "Reconcile does nothing if task doesn't belong to the operation the controller is supervising", + fields: fields{ + NodeName: "foo-node", + Operation: "foo-operation", + Objs: []runtime.Object{ + &operatorv1.Operation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Operation", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "bar-operation", + }, + }, + &operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTaskGroup", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "Operation", + Name: "bar-operation", + }, + }, + }, + }, + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-task", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "RuntimeTaskGroup", + Name: "foo-taskgroup", + }, + }, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-task"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + { + name: "Reconcile pass", + fields: fields{ + NodeName: "foo-node", + Operation: "foo-operation", + Objs: []runtime.Object{ + &operatorv1.Operation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Operation", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-operation", + }, + }, + &operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTaskGroup", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "Operation", + Name: "foo-operation", + }, + }, + }, + }, + &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-task", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "RuntimeTaskGroup", + Name: "foo-taskgroup", + }, + }, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + }, + }, + }, + }, + args: args{ + req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "foo-task"}}, + }, + want: ctrl.Result{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RuntimeTaskReconciler{ + Client: fake.NewFakeClientWithScheme(setupScheme(), tt.fields.Objs...), + NodeName: tt.fields.NodeName, + Operation: tt.fields.Operation, + recorder: record.NewFakeRecorder(1), + Log: log.Log, + } + got, err := r.Reconcile(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("Reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Reconcile() got = %v, want %v", got, tt.want) + } + }) + } +} + +func fixupWantTask(want *operatorv1.RuntimeTask, got *operatorv1.RuntimeTask) { + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.CreationTimestamp.IsZero() { + want.CreationTimestamp = got.CreationTimestamp + } + + // In case want.ErrorMessage is a marker, replace it with the current error + if want.Status.ErrorMessage != nil && *want.Status.ErrorMessage == "error" && got.Status.ErrorMessage != nil { + want.Status.ErrorMessage = got.Status.ErrorMessage + want.Status.ErrorReason = got.Status.ErrorReason + } + + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.Status.StartTime != nil && want.Status.StartTime.IsZero() && got.Status.StartTime != nil { + want.Status.StartTime = got.Status.StartTime + } + // In case want.CompletionTime is a marker, replace it with the current CompletionTime + if want.Status.CompletionTime != nil && want.Status.CompletionTime.IsZero() && got.Status.CompletionTime != nil { + want.Status.CompletionTime = got.Status.CompletionTime + } +} + +func runtimeTaskStatusErrorPtr(s errors.RuntimeTaskStatusError) *errors.RuntimeTaskStatusError { + return &s +} + +func setupScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + if err := operatorv1.AddToScheme(scheme); err != nil { + panic(err) + } + if err := corev1.AddToScheme(scheme); err != nil { + panic(err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + panic(err) + } + return scheme +} diff --git a/controllers/runtimetask_reconcile.go b/controllers/runtimetask_reconcile.go new file mode 100644 index 0000000..98bbf77 --- /dev/null +++ b/controllers/runtimetask_reconcile.go @@ -0,0 +1,156 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "sort" + + corev1 "k8s.io/api/core/v1" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +// Reconcile Task is implemented by matching current Nodes and desired Task, so the controller +// can determine what is necessary to do next + +// taskReconcileItem defines match between desired Task and the corresponding current Node match. +// supported combinations are: +// - desired existing, current missing (current to be created) +// - desired existing, current existing (current to be operated) +// - desired missing, current existing (invalid) + +type taskReconcileItem struct { + name string + node *corev1.Node + tasks []operatorv1.RuntimeTask +} + +func newTaskGroupChildProxy(node *corev1.Node, tasks ...operatorv1.RuntimeTask) *taskReconcileItem { + var name string + if node != nil { + name = node.Name + } else { + name = tasks[0].Spec.NodeName + } + return &taskReconcileItem{ + name: name, + node: node, + tasks: tasks, + } +} + +type taskReconcileList struct { + all []*taskReconcileItem + invalid []*taskReconcileItem + tobeCreated []*taskReconcileItem + pending []*taskReconcileItem + running []*taskReconcileItem + completed []*taskReconcileItem + failed []*taskReconcileItem +} + +func reconcileTasks(nodes []corev1.Node, tasks *operatorv1.RuntimeTaskList) *taskReconcileList { + // Build an empty match for each desired Task (1 for each node) + // N.B. we are storing matches in a Map so we can match Node and Task by NodeName + matchMap := map[string]*taskReconcileItem{} + for _, n := range nodes { + x := n // copies the node to a local variable in order to avoid it to get overridden at the next iteration + matchMap[x.Name] = newTaskGroupChildProxy(&x) + } + + // Match the current Task with desired Task (1 for each node in scope). + for _, t := range tasks.Items { + // in case a current task has a corresponding desired task, match them + // NB. if there are more that one match, we track this, but this is an inconsistency + // (more that one Task for the same node) + if v, ok := matchMap[t.Spec.NodeName]; ok { + // TODO(fabriziopandini): might be we want to check if the task was exactly the expected task + v.tasks = append(v.tasks, t) + continue + } + + // in case a current task does not have desired task, we track this, but this is an inconsistency + // (a Task does not matching any existing node) + matchMap[t.Spec.NodeName] = newTaskGroupChildProxy(nil, t) + } + + // Transpose the matchMap into a list + matchList := &taskReconcileList{ + all: []*taskReconcileItem{}, + invalid: []*taskReconcileItem{}, + tobeCreated: []*taskReconcileItem{}, + pending: []*taskReconcileItem{}, + running: []*taskReconcileItem{}, + completed: []*taskReconcileItem{}, + failed: []*taskReconcileItem{}, + } + + for _, v := range matchMap { + matchList.all = append(matchList.all, v) + } + + // ensure the list is sorted in a predictable way + sort.Slice(matchList.all, func(i, j int) bool { return matchList.all[i].name < matchList.all[j].name }) + + // Build all the derived views, so we can have a quick glance at tasks in different states + matchList.deriveViews() + + return matchList +} + +func (t *taskReconcileList) deriveViews() { + for _, v := range t.all { + switch { + case v.node != nil: + switch len(v.tasks) { + case 0: + // If there is no Task for a Node, the task has to be created by this controller + t.tobeCreated = append(t.tobeCreated, v) + case 1: + // Failed (and not recovering) + if (v.tasks[0].Status.ErrorReason != nil || v.tasks[0].Status.ErrorMessage != nil) && + (v.tasks[0].Spec.GetTypedTaskRecoveryStrategy() == operatorv1.RuntimeTaskRecoveryUnknownStrategy) { + t.failed = append(t.failed, v) + continue + } + // Completed + if v.tasks[0].Status.CompletionTime != nil { + t.completed = append(t.completed, v) + continue + } + // Running (nb. paused Task or recovering Task fall into this counter) + if v.tasks[0].Status.StartTime != nil { + t.running = append(t.running, v) + continue + } + // Pending + t.pending = append(t.pending, v) + default: + // if there are more that one Task for the same node, this is an invalid condition + // NB. in this case it counts as a single replica, even if there are more than one Task + t.invalid = append(t.invalid, v) + } + case v.node == nil: + // if there is a Task without matching node, this is an invalid condition + t.invalid = append(t.invalid, v) + } + } +} + +func (t *taskReconcileList) activeTasks() int { + return len(t.pending) + len(t.running) +} diff --git a/controllers/runtimetask_reconcile_test.go b/controllers/runtimetask_reconcile_test.go new file mode 100644 index 0000000..a2dbd38 --- /dev/null +++ b/controllers/runtimetask_reconcile_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func Test_newTaskGroupChildProxy(t *testing.T) { + n1 := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + } + + t1 := operatorv1.RuntimeTask{ + ObjectMeta: metav1.ObjectMeta{ + Name: "task1", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "test", + }, + } + + type args struct { + node *corev1.Node + tasks []operatorv1.RuntimeTask + } + tests := []struct { + name string + args args + want *taskReconcileItem + }{ + { + name: "Node and task exist", + args: args{ + node: n1, + tasks: []operatorv1.RuntimeTask{t1}, + }, + want: &taskReconcileItem{ + name: "test", + node: n1, + tasks: []operatorv1.RuntimeTask{t1}, + }, + }, + { + name: "Node exists, task does not exist", + args: args{ + node: n1, + tasks: nil, + }, + want: &taskReconcileItem{ + name: "test", + node: n1, + tasks: nil, + }, + }, + { + name: "Node doest not exist, task exists", + args: args{ + node: nil, + tasks: []operatorv1.RuntimeTask{t1}, + }, + want: &taskReconcileItem{ + name: "test", + node: nil, + tasks: []operatorv1.RuntimeTask{t1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := newTaskGroupChildProxy(tt.args.node, tt.args.tasks...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("newTaskGroupChildProxy() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_reconcileTasks(t *testing.T) { + nodes := []corev1.Node{ + { // a node with a matching task/pending + ObjectMeta: metav1.ObjectMeta{ + Name: "node1-a", + }, + }, + { // a node with a matching task/running + ObjectMeta: metav1.ObjectMeta{ + Name: "node1-b", + }, + }, + { // a node with a matching task/completed + ObjectMeta: metav1.ObjectMeta{ + Name: "node1-c", + }, + }, + { // a node with a matching task/failed + ObjectMeta: metav1.ObjectMeta{ + Name: "node1-d", + }, + }, + { // a node without a matching task (task to be created) + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + { // a node with two matching tasks (invalid) + ObjectMeta: metav1.ObjectMeta{ + Name: "node3", + }, + }, + } + tasks := &operatorv1.RuntimeTaskList{ + Items: []operatorv1.RuntimeTask{ + { // a pending task matching node 1 + ObjectMeta: metav1.ObjectMeta{ + Name: "task1-a", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node1-a", + }, + }, + { // a running task matching node 1 + ObjectMeta: metav1.ObjectMeta{ + Name: "task1-b", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node1-b", + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + { // a completed task matching node 1 + ObjectMeta: metav1.ObjectMeta{ + Name: "task1-c", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node1-c", + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + CompletionTime: timePtr(metav1.Now()), + }, + }, + { // a failed task matching node 1 + ObjectMeta: metav1.ObjectMeta{ + Name: "task1-d", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node1-d", + }, + Status: operatorv1.RuntimeTaskStatus{ + StartTime: timePtr(metav1.Now()), + ErrorMessage: stringPtr("error"), + }, + }, + { // a task matching node 3 + ObjectMeta: metav1.ObjectMeta{ + Name: "task2", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node3", + }, + }, + { // another task matching node 3 + ObjectMeta: metav1.ObjectMeta{ + Name: "task3", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node3", + }, + }, + { // a task not matching any node (invalid) + ObjectMeta: metav1.ObjectMeta{ + Name: "task4", + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "node4", + }, + }, + }, + } + + got := reconcileTasks(nodes, tasks) + + // All + want := []string{"node1-a", "node1-b", "node1-c", "node1-d", "node2", "node3", "node4"} + if got := taskNames(got.all); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().all = %v, want %v", got, want) + } + + // pending + want = []string{"node1-a"} + if got := taskNames(got.pending); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().pending = %v, want %v", got, want) + } + + // running + want = []string{"node1-b"} + if got := taskNames(got.running); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().running = %v, want %v", got, want) + } + + // completed + want = []string{"node1-c"} + if got := taskNames(got.completed); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().completed = %v, want %v", got, want) + } + + // failed + want = []string{"node1-d"} + if got := taskNames(got.failed); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().failed = %v, want %v", got, want) + } + + // tobeCreated + want = []string{"node2"} + if got := taskNames(got.tobeCreated); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().tobeCreated = %v, want %v", got, want) + } + + // invalid + want = []string{"node3", "node4"} + if got := taskNames(got.invalid); !reflect.DeepEqual(got, want) { + t.Errorf("newTaskGroupChildProxy().invalid = %v, want %v", got, want) + } +} + +func taskNames(got []*taskReconcileItem) []string { + var actual []string + for _, a := range got { + actual = append(actual, a.name) + } + return actual +} + +func timePtr(t metav1.Time) *metav1.Time { + return &t +} + +func stringPtr(s string) *string { + return &s +} + +func boolPtr(s bool) *bool { + return &s +} diff --git a/controllers/runtimetaskgroup_controller.go b/controllers/runtimetaskgroup_controller.go new file mode 100644 index 0000000..6a88f71 --- /dev/null +++ b/controllers/runtimetaskgroup_controller.go @@ -0,0 +1,358 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "strings" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" + operatorerrors "k8s.io/kubeadm/operator/errors" +) + +// RuntimeTaskGroupReconciler reconciles a RuntimeTaskGroup object +type RuntimeTaskGroupReconciler struct { + client.Client + recorder record.EventRecorder + Log logr.Logger +} + +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=runtimetaskgroups,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.kubeadm.x-k8s.io,resources=runtimetaskgroups/status,verbs=get;update;patch + +// SetupWithManager configures the controller for calling the reconciler +func (r *RuntimeTaskGroupReconciler) SetupWithManager(mgr ctrl.Manager) error { + var mapFunc handler.ToRequestsFunc = func(o handler.MapObject) []reconcile.Request { + return operationToTaskGroupRequests(r.Client, o) + } + + err := ctrl.NewControllerManagedBy(mgr). + For(&operatorv1.RuntimeTaskGroup{}). + Owns(&operatorv1.RuntimeTask{}). // force reconcile TaskGroup every time one of the owned TaskGroups change + Watches( // force reconcile TaskGroup every time the parent operation changes + &source.Kind{Type: &operatorv1.Operation{}}, + &handler.EnqueueRequestsFromMapFunc{ToRequests: mapFunc}, + ). + Complete(r) + + r.recorder = mgr.GetEventRecorderFor("runtime-taskgroup-controller") + return err +} + +// Reconcile a runtimetaskgroup +func (r *RuntimeTaskGroupReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr error) { + ctx := context.Background() + log := r.Log.WithValues("task-group", req.NamespacedName) + + // Fetch the TaskGroup instance + taskgroup := &operatorv1.RuntimeTaskGroup{} + if err := r.Client.Get(ctx, req.NamespacedName, taskgroup); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Ignore the TaskGroup if it is already completed or failed + if taskgroup.Status.CompletionTime != nil { + return ctrl.Result{}, nil + } + + // Fetch the Operation instance + operation, err := getOwnerOperation(ctx, r.Client, taskgroup.ObjectMeta) + if err != nil { + return ctrl.Result{}, err + } + + // Initialize the patch helper + patchHelper, err := patch.NewHelper(taskgroup, r) + if err != nil { + return ctrl.Result{}, err + } + // Always attempt to Patch the TaskGroup object and status after each reconciliation. + defer func() { + if err := patchHelper.Patch(ctx, taskgroup); err != nil { + log.Error(err, "failed to patch TaskGroup") + if rerr == nil { + rerr = err + } + } + }() + + // Reconcile the TaskGroup + if err := r.reconcileTaskGroup(operation, taskgroup, log); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +func (r *RuntimeTaskGroupReconciler) reconcileTaskGroup(operation *operatorv1.Operation, taskgroup *operatorv1.RuntimeTaskGroup, log logr.Logger) (err error) { + // gets relevant settings from top level objects + executionMode := operation.Spec.GetTypedOperationExecutionMode() + operationPaused := operation.Status.Paused + + // Reconcile paused override from top level objects + r.reconcilePauseOverride(operationPaused, taskgroup) + + // Handle deleted TaskGroup + if !taskgroup.DeletionTimestamp.IsZero() { + err = r.reconcileDelete(taskgroup) + if err != nil { + return err + } + } + // Handle non-deleted TaskGroup + + // gets controlled tasks items (desired vs actual) + tasks, err := r.reconcileTasks(executionMode, taskgroup, log) + if err != nil { + return err + } + + err = r.reconcileNormal(executionMode, taskgroup, tasks, log) + if err != nil { + return err + } + + // Always reconcile Phase at the end + r.reconcilePhase(taskgroup) + + return nil +} + +func (r *RuntimeTaskGroupReconciler) reconcilePauseOverride(operationPaused bool, taskgroup *operatorv1.RuntimeTaskGroup) { + // record paused override state change, if any + taskgrouppaused := operationPaused + recordPausedChange(r.recorder, taskgroup, taskgroup.Status.Paused, taskgrouppaused, "by top level objects") + + // update status with paused override setting from top level objects + taskgroup.Status.Paused = taskgrouppaused +} + +func (r *RuntimeTaskGroupReconciler) reconcileTasks(executionMode operatorv1.OperationExecutionMode, taskgroup *operatorv1.RuntimeTaskGroup, log logr.Logger) (*taskReconcileList, error) { + // gets all the Node object matching the taskgroup.Spec.NodeSelector + // those are the Node where the task taskgroup.Spec.Template should be replicated (desired tasks) + nodes, err := listNodesBySelector(r.Client, &taskgroup.Spec.NodeSelector) + if err != nil { + return nil, errors.Wrap(err, "failed to list nodes") + } + + desired := filterNodes(nodes, taskgroup.Spec.GetTypedTaskGroupNodeFilter()) + + // gets all the Task objects matching the taskgroup.Spec.Selector. + // those are the current Task objects controlled by this deployment + current, err := listTasksBySelector(r.Client, &taskgroup.Spec.Selector) + if err != nil { + return nil, errors.Wrap(err, "failed to list tasks") + } + + log.Info("reconciling", "Nodes", len(desired), "Tasks", len(current.Items)) + + // match current and desired state, so the controller can determine what is necessary to do next + tasks := reconcileTasks(desired, current) + + // update replica counters + taskgroup.Status.Nodes = int32(len(tasks.all)) + taskgroup.Status.RunningNodes = int32(len(tasks.running)) + taskgroup.Status.SucceededNodes = int32(len(tasks.completed)) + taskgroup.Status.FailedNodes = int32(len(tasks.failed)) + taskgroup.Status.InvalidNodes = int32(len(tasks.invalid)) + + return tasks, nil +} + +func (r *RuntimeTaskGroupReconciler) reconcileNormal(executionMode operatorv1.OperationExecutionMode, taskgroup *operatorv1.RuntimeTaskGroup, tasks *taskReconcileList, log logr.Logger) error { + // If the TaskGroup doesn't have finalizer, add it. + //if !util.Contains(taskgroup.Finalizers, operatorv1alpha1.TaskGroupFinalizer) { + // taskgroup.Finalizers = append(taskgroup.Finalizers, operatorv1alpha1.TaskGroupFinalizer) + //} + + // If there are Tasks not yet completed (pending or running), cleanup error messages (required e.g. after recovery) + // NB. It is necessary to give priority to running vs errors so the operation controller keeps alive/restarts + // the DaemonsSet for processing tasks + if tasks.activeTasks() > 0 { + taskgroup.Status.ResetError() + } else { + // if there are invalid combinations (e.g. a Node with more than one Task, or a Task without a Node), + // set the error and stop creating new Tasks + if len(tasks.invalid) > 0 { + taskgroup.Status.SetError( + operatorerrors.NewRuntimeTaskGroupReconciliationError("something invalid"), + ) + return nil + } + + // if there are failed tasks + // set the error and stop creating new Tasks + if len(tasks.failed) > 0 { + taskgroup.Status.SetError( + operatorerrors.NewRuntimeTaskGroupReplicaError("something failed"), + ) + return nil + } + } + + // TODO: manage adopt tasks/tasks to be orphaned + + // if nil, set the TaskGroup start time + if taskgroup.Status.StartTime == nil { + taskgroup.Status.SetStartTime() + + //TODO: add a signature so we can detect if someone/something changes the taskgroup while it is processed + + return nil + } + + // if the completed Task have reached the number of expected Task, the TaskGroup is completed + // NB. we are doing this before checking pause because if everything is completed, does not make sense to pause + if len(tasks.completed) == len(tasks.all) { + // NB. we are setting this condition explicitly in order to avoid that the taskGroup accidentally + // restarts to create tasks + taskgroup.Status.SetCompletionTime() + return nil + } + + // if the TaskGroup is paused, return + if taskgroup.Status.Paused { + return nil + } + + // otherwise, proceed creating tasks + + // if there are still Tasks to be created + if len(tasks.tobeCreated) > 0 { + //TODO: manage different deployment strategy e.g. parallel + + // if there no existing Tasks not yet completed (pending or running) + if tasks.activeTasks() == 0 { + // create a Task for the next node in the ordered sequence + nextNode := tasks.tobeCreated[0].node.Name + log.WithValues("node-name", nextNode).Info("creating task") + + err := r.createTasksReplica(executionMode, taskgroup, nextNode) + if err != nil { + if strings.Contains(err.Error(), "already exists") { + log.WithValues("node-name", nextNode).Info("task already exists") + return nil + } + return errors.Wrap(err, "Failed to create Task replica") + } + } + } + + return nil +} + +func (r *RuntimeTaskGroupReconciler) createTasksReplica(executionMode operatorv1.OperationExecutionMode, taskgroup *operatorv1.RuntimeTaskGroup, nodeName string) error { + r.Log.Info("Creating task replica", "node", nodeName) + + gv := operatorv1.GroupVersion + + paused := false + if executionMode == operatorv1.OperationExecutionModeControlled { + paused = true + } + + // todo: use a template instead of a new object; currently the template labels are not set successfully + r.Log.Info("template logs", "taskgroup", taskgroup, "labels", taskgroup.Spec.Template.GetObjectMeta().GetLabels()) + labels := taskgroup.Spec.Template.Labels + if len(labels) == 0 { + labels = taskgroup.Labels + } + + task := &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: gv.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", taskgroup.Name, nodeName), //TODO: GeneratedName? + Namespace: taskgroup.Namespace, + // we should use the same labels as the taskgroup template + Labels: labels, + Annotations: taskgroup.Spec.Template.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(taskgroup, taskgroup.GroupVersionKind())}, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: nodeName, + Commands: taskgroup.Spec.Template.Spec.Commands, + }, + Status: operatorv1.RuntimeTaskStatus{ + Phase: string(operatorv1.RuntimeTaskPhasePending), + Paused: paused, + }, + } + + return r.Client.Create(context.Background(), task) +} + +func (r *RuntimeTaskGroupReconciler) reconcileDelete(taskgroup *operatorv1.RuntimeTaskGroup) error { + + // TaskGroup is deleted so remove the finalizer. + //taskgroup.Finalizers = util.Filter(taskgroup.Finalizers, operatorv1alpha1.TaskGroupFinalizer) + + return nil +} + +func (r *RuntimeTaskGroupReconciler) reconcilePhase(taskgroup *operatorv1.RuntimeTaskGroup) { + // Set the phase to "deleting" if the deletion timestamp is set. + if !taskgroup.DeletionTimestamp.IsZero() { + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhaseDeleted) + return + } + + // Set the phase to "failed" if any of Status.ErrorReason or Status.ErrorMessage is not nil. + if taskgroup.Status.ErrorReason != nil || taskgroup.Status.ErrorMessage != nil { + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhaseFailed) + return + } + + // Set the phase to "succeeded" if completion date is set. + if taskgroup.Status.CompletionTime != nil { + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhaseSucceeded) + return + } + + // Set the phase to "paused" if paused set. + if taskgroup.Status.Paused { + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhasePaused) + return + } + + // Set the phase to "running" if start date is set. + if taskgroup.Status.StartTime != nil { + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhaseRunning) + return + } + + // Set the phase to "pending". + taskgroup.Status.SetTypedPhase(operatorv1.RuntimeTaskGroupPhasePending) +} diff --git a/controllers/runtimetaskgroup_controller_test.go b/controllers/runtimetaskgroup_controller_test.go new file mode 100644 index 0000000..80d378e --- /dev/null +++ b/controllers/runtimetaskgroup_controller_test.go @@ -0,0 +1,711 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func TestRuntimeTaskGorupReconcilePhase(t *testing.T) { + tx := metav1.Now() + errMessage := "error" + + tests := []struct { + name string + input *operatorv1.RuntimeTaskGroup + expected *operatorv1.RuntimeTaskGroup + }{ + { + name: "Reconcile pending state", + input: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{}, + }, + expected: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Phase: string(operatorv1.RuntimeTaskGroupPhasePending), + }, + }, + }, + { + name: "Reconcile running state", + input: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + }, + }, + expected: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + Phase: string(operatorv1.RuntimeTaskGroupPhaseRunning), + }, + }, + }, + { + name: "Reconcile paused state", + input: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + Paused: true, + }, + }, + expected: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + Paused: true, + Phase: string(operatorv1.RuntimeTaskGroupPhasePaused), + }, + }, + }, + { + name: "Reconcile succeeded state", + input: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + CompletionTime: &tx, + }, + }, + expected: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + CompletionTime: &tx, + Phase: string(operatorv1.RuntimeTaskGroupPhaseSucceeded), + }, + }, + }, + { + name: "Reconcile failed state", + input: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + ErrorMessage: &errMessage, + }, + }, + expected: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + ErrorMessage: &errMessage, + Phase: string(operatorv1.RuntimeTaskGroupPhaseFailed), + }, + }, + }, + { + name: "Reconcile deleted state", + input: &operatorv1.RuntimeTaskGroup{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &tx, + }, + }, + expected: &operatorv1.RuntimeTaskGroup{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &tx, + }, + Status: operatorv1.RuntimeTaskGroupStatus{ + Phase: string(operatorv1.RuntimeTaskGroupPhaseDeleted), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RuntimeTaskGroupReconciler{} + r.reconcilePhase(tt.input) + + if !reflect.DeepEqual(tt.input, tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, tt.input) + } + }) + } +} + +func TestRuntimeTaskGorupReconcilePauseOverride(t *testing.T) { + type input struct { + operationPaused bool + taskgroup *operatorv1.RuntimeTaskGroup + } + type expected struct { + taskgroup *operatorv1.RuntimeTaskGroup + events int + } + + tests := []struct { + name string + input input + expected expected + }{ + { + name: "Reconcile a RuntimeTaskGroup not paused with an Operation not paused", + input: input{ + operationPaused: false, + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: false, + }, + }, + }, + expected: expected{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: false, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a RuntimeTaskGroup not paused with an Operation paused", + input: input{ + operationPaused: true, + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: false, + }, + }, + }, + expected: expected{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: true, + }, + }, + events: 1, + }, + }, + { + name: "Reconcile a RuntimeTaskGroup paused with an Operation paused", + input: input{ + operationPaused: true, + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: true, + }, + }, + }, + expected: expected{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: true, + }, + }, + events: 0, + }, + }, + { + name: "Reconcile a RuntimeTaskGroup paused with an Operation not paused", + input: input{ + operationPaused: false, + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: true, + }, + }, + }, + expected: expected{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + Paused: false, + }, + }, + events: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := record.NewFakeRecorder(1) + + r := &RuntimeTaskGroupReconciler{ + recorder: rec, + } + + r.reconcilePauseOverride(tt.input.operationPaused, tt.input.taskgroup) + + if !reflect.DeepEqual(tt.input.taskgroup, tt.expected.taskgroup) { + t.Errorf("expected %v, got %v", tt.expected.taskgroup, tt.input.taskgroup) + } + + if tt.expected.events != len(rec.Events) { + t.Errorf("expected %v, got %v", tt.expected.events, len(rec.Events)) + } + }) + } +} + +func TestRuntimeTaskGroupReconciler_createTasksReplica(t *testing.T) { + type args struct { + executionMode operatorv1.OperationExecutionMode + taskgroup *operatorv1.RuntimeTaskGroup + nodeName string + } + tests := []struct { + name string + args args + want *operatorv1.RuntimeTask + wantErr bool + }{ + { + name: "Create a task", + args: args{ + executionMode: operatorv1.OperationExecutionModeAuto, + taskgroup: &operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTaskGroup", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup", + }, + Spec: operatorv1.RuntimeTaskGroupSpec{ + Template: operatorv1.RuntimeTaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"foo-label": "foo"}, + Annotations: map[string]string{"foo-annotation": "foo"}, + }, + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + }, + }, + }, + nodeName: "foo-node", + }, + want: &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup-foo-node", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "RuntimeTaskGroup", + Name: "foo-taskgroup", + UID: "", + Controller: boolPtr(true), + BlockOwnerDeletion: boolPtr(true), + }, + }, + CreationTimestamp: metav1.Time{}, //using zero as a marker for "whatever time it was created" + Labels: map[string]string{"foo-label": "foo"}, + Annotations: map[string]string{"foo-annotation": "foo"}, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + Phase: string(operatorv1.RuntimeTaskPhasePending), + Paused: false, + }, + }, + wantErr: false, + }, + { + name: "Create a paused task if execution mode=Controlled", + args: args{ + executionMode: operatorv1.OperationExecutionModeControlled, + taskgroup: &operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTaskGroup", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup", + }, + Spec: operatorv1.RuntimeTaskGroupSpec{ + Template: operatorv1.RuntimeTaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"foo-label": "foo"}, + Annotations: map[string]string{"foo-annotation": "foo"}, + }, + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + }, + }, + }, + nodeName: "foo-node", + }, + want: &operatorv1.RuntimeTask{ + TypeMeta: metav1.TypeMeta{ + Kind: "RuntimeTask", + APIVersion: operatorv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo-taskgroup-foo-node", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: operatorv1.GroupVersion.String(), + Kind: "RuntimeTaskGroup", + Name: "foo-taskgroup", + UID: "", + Controller: boolPtr(true), + BlockOwnerDeletion: boolPtr(true), + }, + }, + CreationTimestamp: metav1.Time{}, //using zero as a marker for "whatever time it was created" + Labels: map[string]string{"foo-label": "foo"}, + Annotations: map[string]string{"foo-annotation": "foo"}, + }, + Spec: operatorv1.RuntimeTaskSpec{ + NodeName: "foo-node", + Commands: []operatorv1.CommandDescriptor{ + {}, + }, + }, + Status: operatorv1.RuntimeTaskStatus{ + Phase: string(operatorv1.RuntimeTaskPhasePending), + Paused: true, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RuntimeTaskGroupReconciler{ + Client: fake.NewFakeClientWithScheme(setupScheme()), + Log: log.Log, + } + if err := r.createTasksReplica(tt.args.executionMode, tt.args.taskgroup, tt.args.nodeName); (err != nil) != tt.wantErr { + t.Errorf("createTasksReplica() error = %v, wantErr %v", err, tt.wantErr) + } + + got := &operatorv1.RuntimeTask{} + key := client.ObjectKey{ + Namespace: tt.want.Namespace, + Name: tt.want.Name, + } + if err := r.Client.Get(context.Background(), key, got); err != nil { + t.Errorf("Get() error = %v", err) + return + } + + fixupWantTask(tt.want, got) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("createTasksReplica() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRuntimeTaskGroupReconciler_reconcileNormal(t *testing.T) { + type args struct { + executionMode operatorv1.OperationExecutionMode + taskgroup *operatorv1.RuntimeTaskGroup + tasks *taskReconcileList + } + type want struct { + taskgroup *operatorv1.RuntimeTaskGroup + } + tests := []struct { + name string + args args + want want + wantTasks int + wantErr bool + }{ + { + name: "Reconcile sets error if a task is failed and no task is active", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{}, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + }, + failed: []*taskReconcileItem{ + {}, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + ErrorMessage: stringPtr("error"), //using error as a marker for "whatever error was raised" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile sets error if a task is invalid and no task is active", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{}, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + }, + invalid: []*taskReconcileItem{ + {}, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + ErrorMessage: stringPtr("error"), //using error as a marker for "whatever error was raised" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile set start time", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{}, + tasks: &taskReconcileList{}, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile reset error if a task is active", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + ErrorMessage: stringPtr("error"), + }, + }, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + {}, + }, + running: []*taskReconcileItem{ + {}, + }, + failed: []*taskReconcileItem{ // failed should be ignored if a task is active + {}, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile set completion time if no more task to create", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + tasks: &taskReconcileList{}, //empty list of nodes -> no more task to create + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + CompletionTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it completed" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile do nothing if paused", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Now()), + Paused: true, + }, + }, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + }, + tobeCreated: []*taskReconcileItem{ + {}, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + Paused: true, + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + { + name: "Reconcile creates a task if nothing running", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + }, + tobeCreated: []*taskReconcileItem{ + { + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + }, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTasks: 1, + wantErr: false, + }, + { + name: "Reconcile does not creates a task if something running", + args: args{ + executionMode: "", + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Now()), + }, + }, + tasks: &taskReconcileList{ + all: []*taskReconcileItem{ + {}, + {}, + }, + tobeCreated: []*taskReconcileItem{ + {}, + }, + running: []*taskReconcileItem{ + {}, + }, + }, + }, + want: want{ + taskgroup: &operatorv1.RuntimeTaskGroup{ + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: timePtr(metav1.Time{}), //using zero as a marker for "whatever time it started" + }, + }, + }, + wantTasks: 0, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := fake.NewFakeClientWithScheme(setupScheme()) + + r := &RuntimeTaskGroupReconciler{ + Client: c, + recorder: record.NewFakeRecorder(1), + Log: log.Log, + } + if err := r.reconcileNormal(tt.args.executionMode, tt.args.taskgroup, tt.args.tasks, r.Log); (err != nil) != tt.wantErr { + t.Errorf("reconcileNormal() error = %v, wantErr %v", err, tt.wantErr) + } + + fixupWantTaskGroup(tt.want.taskgroup, tt.args.taskgroup) + + if !reflect.DeepEqual(tt.args.taskgroup, tt.want.taskgroup) { + t.Errorf("reconcileNormal() = %v, want %v", tt.args.taskgroup, tt.want.taskgroup) + } + + tasks := &operatorv1.RuntimeTaskList{} + if err := c.List(context.Background(), tasks); err != nil { + t.Fatalf("List() error = %v", err) + } + + if len(tasks.Items) != tt.wantTasks { + t.Errorf("reconcileNormal() = %v tasks, want %v tasks", len(tasks.Items), tt.wantTasks) + } + }) + } +} + +func fixupWantTaskGroup(want *operatorv1.RuntimeTaskGroup, got *operatorv1.RuntimeTaskGroup) { + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.CreationTimestamp.IsZero() { + want.CreationTimestamp = got.CreationTimestamp + } + + // In case want.ErrorMessage is a marker, replace it with the current error + if want.Status.ErrorMessage != nil && *want.Status.ErrorMessage == "error" && got.Status.ErrorMessage != nil { + want.Status.ErrorMessage = got.Status.ErrorMessage + want.Status.ErrorReason = got.Status.ErrorReason + } + + // In case want.StartTime is a marker, replace it with the current CompletionTime + if want.Status.StartTime != nil && want.Status.StartTime.IsZero() && got.Status.StartTime != nil { + want.Status.StartTime = got.Status.StartTime + } + // In case want.CompletionTime is a marker, replace it with the current CompletionTime + if want.Status.CompletionTime != nil && want.Status.CompletionTime.IsZero() && got.Status.CompletionTime != nil { + want.Status.CompletionTime = got.Status.CompletionTime + } +} diff --git a/controllers/runtimetaskgroup_reconcile.go b/controllers/runtimetaskgroup_reconcile.go new file mode 100644 index 0000000..3f24cca --- /dev/null +++ b/controllers/runtimetaskgroup_reconcile.go @@ -0,0 +1,147 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "sort" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +// Reconcile TaskGroups is implemented by matching current and desired TaskGroup, so the controller +// can determine what is necessary to do next + +// taskGroupReconcileItem defines match between desired TaskGroup and the corresponding current TaskGroup match. +// supported combinations are: +// - desired existing, current missing (current to be created) +// - desired existing, current existing (current to be operated) +// - desired missing, current existing (invalid) +type taskGroupReconcileItem struct { + name string + planned *operatorv1.RuntimeTaskGroup + current *operatorv1.RuntimeTaskGroup +} + +func newTaskGroupReconcileItem(planned *operatorv1.RuntimeTaskGroup, current *operatorv1.RuntimeTaskGroup) *taskGroupReconcileItem { + var name string + if planned != nil { + name = planned.Name + } else { + name = current.Name + } + return &taskGroupReconcileItem{ + name: name, + planned: planned, + current: current, + } +} + +type taskGroupReconcileList struct { + all []*taskGroupReconcileItem + invalid []*taskGroupReconcileItem + tobeCreated []*taskGroupReconcileItem + pending []*taskGroupReconcileItem + running []*taskGroupReconcileItem + completed []*taskGroupReconcileItem + failed []*taskGroupReconcileItem +} + +func reconcileTaskGroups(desired *operatorv1.RuntimeTaskGroupList, current *operatorv1.RuntimeTaskGroupList) *taskGroupReconcileList { + // Build an empty match for each desired TaskGroup + // N.B. we are storing matches in a Map so we can match desired and current TaskGroup by Name + matchMap := map[string]*taskGroupReconcileItem{} + for _, taskGroup := range desired.Items { + desiredTaskGroup := taskGroup // copies the desired TaskGroup to a local variable in order to avoid it to get overridden at the next iteration + matchMap[desiredTaskGroup.Name] = newTaskGroupReconcileItem(&desiredTaskGroup, nil) + } + + // Match the current child objects (TaskGroup) with desired objects (desired TaskGroup). + for _, taskGroup := range current.Items { + currentTaskGroup := taskGroup // copies the TaskGroup to a local variable in order to avoid it to get overridden at the next iteration + // in case a current objects has a corresponding desired object, match them + // NB. if there are more that one match, we track this, but this is an inconsistency + // (more that one Task for the same node) + if v, ok := matchMap[currentTaskGroup.Name]; ok { + // TODO: might be we want to check if the task was exactly the expected task + v.current = ¤tTaskGroup + continue + } + + // in case a current objects does not have desired object, we track this, but this is an inconsistency + // (a TaskGroup does not matching any desired TaskGroup) + matchMap[currentTaskGroup.Name] = newTaskGroupReconcileItem(nil, ¤tTaskGroup) + } + + // Transpose the childMap into a list + matchList := &taskGroupReconcileList{ + all: []*taskGroupReconcileItem{}, + invalid: []*taskGroupReconcileItem{}, + tobeCreated: []*taskGroupReconcileItem{}, + pending: []*taskGroupReconcileItem{}, + running: []*taskGroupReconcileItem{}, + completed: []*taskGroupReconcileItem{}, + failed: []*taskGroupReconcileItem{}, + } + + for _, v := range matchMap { + matchList.all = append(matchList.all, v) + } + + // ensure the list is sorted in a predictable way + sort.Slice(matchList.all, func(i, j int) bool { return matchList.all[i].name < matchList.all[j].name }) + + // Build all the derived views, so we can have a quick glance at taskGroups in different states + matchList.deriveViews() + + return matchList +} + +func (a *taskGroupReconcileList) deriveViews() { + for _, v := range a.all { + switch { + case v.planned != nil: + // If there is not TaskGroup for a desired TaskGroup, the TaskGroup has to be created by this controller + if v.current == nil { + a.tobeCreated = append(a.tobeCreated, v) + continue + } + // Failed + if v.current.Status.ErrorReason != nil || v.current.Status.ErrorMessage != nil { + a.failed = append(a.failed, v) + continue + } + // Completed + if v.current.Status.CompletionTime != nil { + a.completed = append(a.completed, v) + continue + } + // Running (nb. paused TaskGroup fall into this counter) + if v.current.Status.StartTime != nil { + a.running = append(a.running, v) + continue + } + // Pending + a.pending = append(a.pending, v) + case v.planned == nil: + a.invalid = append(a.invalid, v) + } + } +} + +func (a *taskGroupReconcileList) activeTaskGroups() int { + return len(a.pending) + len(a.running) +} diff --git a/controllers/runtimetaskgroup_reconcile_test.go b/controllers/runtimetaskgroup_reconcile_test.go new file mode 100644 index 0000000..b064187 --- /dev/null +++ b/controllers/runtimetaskgroup_reconcile_test.go @@ -0,0 +1,221 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func TestNewTaskGroupReconcileItem(t *testing.T) { + type input struct { + planned *operatorv1.RuntimeTaskGroup + current *operatorv1.RuntimeTaskGroup + } + + planned := &operatorv1.RuntimeTaskGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "planned", + }, + } + current := &operatorv1.RuntimeTaskGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "current", + }, + } + + tests := []struct { + name string + input input + expected *taskGroupReconcileItem + }{ + { + name: "Planed and current RuntimeTaskGroup exist", + input: input{ + planned: planned, + current: current, + }, + expected: &taskGroupReconcileItem{ + name: "planned", + planned: planned, + current: current, + }, + }, + { + name: "Planed RuntimeTaskGroup exists, current RuntimeTaskGroup does not exist", + input: input{ + planned: planned, + current: nil, + }, + expected: &taskGroupReconcileItem{ + name: "planned", + planned: planned, + current: nil, + }, + }, + { + name: "Planed RuntimeTaskGroupdoes not exist, current RuntimeTaskGroup exists", + input: input{ + planned: nil, + current: current, + }, + expected: &taskGroupReconcileItem{ + name: "current", + planned: nil, + current: current, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := newTaskGroupReconcileItem(tt.input.planned, tt.input.current); !reflect.DeepEqual(got, tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, got) + } + }) + } +} + +func TestReconcileTaskGroups(t *testing.T) { + tx := metav1.Now() + errMessage := "error" + + desired := &operatorv1.RuntimeTaskGroupList{ + Items: []operatorv1.RuntimeTaskGroup{ + { // there is not current RuntimeTaskGroup for a desired RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-a", + }, + }, + { // a planned RuntimeTaskGroup with a matching current RuntimeTaskGroup/failed + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-b", + }, + }, + { // a planned RuntimeTaskGroup with a matching current RuntimeTaskGroup/completed + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-c", + }, + }, + { // a planned RuntimeTaskGroup with a matching current RuntimeTaskGroup/running + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-d", + }, + }, + { // a planned RuntimeTaskGroup with a matching current RuntimeTaskGroup/pending + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-e", + }, + }, + }, + } + current := &operatorv1.RuntimeTaskGroupList{ + Items: []operatorv1.RuntimeTaskGroup{ + { // a failed current RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-b", + }, + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + ErrorMessage: &errMessage, + }, + }, + { // a completed current RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-c", + }, + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + CompletionTime: &tx, + }, + }, + { // a running current RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-d", + }, + Status: operatorv1.RuntimeTaskGroupStatus{ + StartTime: &tx, + }, + }, + { // a pending current RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-e", + }, + }, + { // a current RuntimeTaskGroup not matching any desired RuntimeTaskGroup + ObjectMeta: metav1.ObjectMeta{ + Name: "taskgroup-f", + }, + }, + }, + } + + got := reconcileTaskGroups(desired, current) + + // All + expected := []string{"taskgroup-a", "taskgroup-b", "taskgroup-c", "taskgroup-d", "taskgroup-e", "taskgroup-f"} + if got := taskGroupNames(got.all); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // pending + expected = []string{"taskgroup-e"} + if got := taskGroupNames(got.pending); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // running + expected = []string{"taskgroup-d"} + if got := taskGroupNames(got.running); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // completed + expected = []string{"taskgroup-c"} + if got := taskGroupNames(got.completed); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // failed + expected = []string{"taskgroup-b"} + if got := taskGroupNames(got.failed); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // tobeCreated + expected = []string{"taskgroup-a"} + if got := taskGroupNames(got.tobeCreated); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } + + // invalid + expected = []string{"taskgroup-f"} + if got := taskGroupNames(got.invalid); !reflect.DeepEqual(got, expected) { + t.Errorf("expected %v, got %v", expected, got) + } +} + +func taskGroupNames(got []*taskGroupReconcileItem) []string { + var actual []string + for _, a := range got { + actual = append(actual, a.name) + } + return actual +} diff --git a/controllers/util.go b/controllers/util.go new file mode 100644 index 0000000..f6c397d --- /dev/null +++ b/controllers/util.go @@ -0,0 +1,436 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" + "k8s.io/kubeadm/operator/operations" +) + +func getImage(c client.Client, namespace, name string) (string, error) { + pod := &corev1.Pod{} + key := client.ObjectKey{ + Namespace: namespace, + Name: name, + } + err := c.Get( + context.Background(), key, pod, + ) + if err != nil { + return "", errors.Errorf("error reading pod %s/%s", namespace, name) + } + + var managerImage string + for _, c := range pod.Spec.Containers { + if c.Name == "manager" { + managerImage = c.Image + } + } + + if managerImage == "" { + return "", errors.Errorf("unable to get Image for manager container in %s/%s", namespace, name) + } + + return managerImage, nil +} + +func getDaemonSet(c client.Client, operation *operatorv1.Operation, namespace string) (*appsv1.DaemonSet, error) { + daemonSet := &appsv1.DaemonSet{} + key := client.ObjectKey{ + Namespace: namespace, + Name: daemonSetName(operation.Name), + } + err := c.Get( + context.Background(), key, daemonSet, + ) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + return daemonSet, nil +} + +func daemonSetName(operationName string) string { + return fmt.Sprintf("controller-agent-%s", operationName) +} + +func hostPathTypePtr(t corev1.HostPathType) *corev1.HostPathType { + return &t +} + +func createDaemonSet(c client.Client, operation *operatorv1.Operation, namespace, image string, metricsRBAC bool) error { + labels := map[string]string{} + for k, v := range operation.Labels { + labels[k] = v + } + + daemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: daemonSetName(operation.Name), + Labels: labels, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(operation, operation.GroupVersionKind())}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + CreationTimestamp: metav1.Now(), + }, + Spec: corev1.PodSpec{ + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Containers: []corev1.Container{ + { + Name: "agent", + Image: image, + Command: []string{ + "/manager", + }, + Args: []string{ + "--mode=agent", + "--agent-node-name=$(MY_NODE_NAME)", + fmt.Sprintf("--agent-operation=%s", operation.Name), + }, + Env: []corev1.EnvVar{ + { + Name: "MY_NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("30Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.BoolPtr(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kubeadm-binary", + MountPath: "/usr/bin/kubeadm", + }, + { + Name: "etc-kubernetes", + MountPath: "/etc/kubernetes", + }, + }, + }, + }, + TerminationGracePeriodSeconds: pointer.Int64Ptr(10), + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "kubeadm-binary", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/usr/bin/kubeadm", + Type: hostPathTypePtr(corev1.HostPathFile), + }, + }, + }, + { + Name: "etc-kubernetes", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kubernetes", + Type: hostPathTypePtr(corev1.HostPathDirectory), + }, + }, + }, + }, + }, + }, + }, + } + + extraLabels, err := operations.DaemonSetNodeSelectorLabels(operation) + if err != nil { + return errors.Wrapf(err, "failed to get NodeSelector for the operation DaemonSet %s/%s", daemonSet.Namespace, daemonSet.Name) + } + if len(extraLabels) > 0 { + daemonSet.Spec.Template.Spec.NodeSelector = extraLabels + } + + if metricsRBAC { + // Force /metrics to be accessible only locally + daemonSet.Spec.Template.Spec.Containers[0].Args = append(daemonSet.Spec.Template.Spec.Containers[0].Args, + "--metrics-addr=127.0.0.1:8080", + ) + + // Expose /metrics via rbac-proxy sidecar + daemonSet.Spec.Template.Spec.Containers = append(daemonSet.Spec.Template.Spec.Containers, + corev1.Container{ + Name: "kube-rbac-proxy", + Image: "gcr.m.daocloud.io/kubebuilder/kube-rbac-proxy:v0.4.0", + Args: []string{ + "--secure-listen-address=0.0.0.0:8443", + "--upstream=http://127.0.0.1:8080/", + "--logtostderr=true", + "--v=10", + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8443, + Name: "https", + }, + }, + }, + ) + + } else { + // Expose /metrics on default (insecure) port + if daemonSet.Annotations == nil { + daemonSet.Annotations = map[string]string{} + } + daemonSet.Annotations["prometheus.io/scrape"] = "true" + daemonSet.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + ContainerPort: 8080, + Name: "metrics", + Protocol: corev1.ProtocolTCP, + }, + } + } + + if err := c.Create( + context.Background(), daemonSet, + ); err != nil { + return errors.Wrapf(err, "failed to create DaemonSet %s/%s", daemonSet.Namespace, daemonSet.Name) + } + + return nil +} + +func deleteDaemonSet(c client.Client, daemonSet *appsv1.DaemonSet) error { + if err := c.Delete( + context.Background(), daemonSet, + ); err != nil { + return errors.Wrapf(err, "failed to delete DaemonSet %s/%s", daemonSet.Namespace, daemonSet.Name) + } + + return nil +} + +func listTaskGroupsByLabels(c client.Client, labels map[string]string) (*operatorv1.RuntimeTaskGroupList, error) { + taskdeployments := &operatorv1.RuntimeTaskGroupList{} + if err := c.List( + context.Background(), taskdeployments, + client.MatchingLabels(labels), + ); err != nil { + return nil, err + } + + return taskdeployments, nil +} + +func recordPausedChange(recorder record.EventRecorder, obj runtime.Object, current, new bool, args ...string) { + if current != new { + reasonVerb := "Paused" + messageAction := "set to pause" + if !new { + reasonVerb = "Restarted" + messageAction = "set for restart" + } + + reason := fmt.Sprintf("%s%s", obj.GetObjectKind().GroupVersionKind().Kind, reasonVerb) + message := fmt.Sprintf("%s %s", obj.GetObjectKind().GroupVersionKind().Kind, messageAction) + if len(args) > 0 { + message = fmt.Sprintf("%s %s", message, strings.Join(args, " ")) + } + recorder.Event(obj, corev1.EventTypeNormal, reason, message) + } +} + +func operationToTaskGroupRequests(c client.Client, o handler.MapObject) []ctrl.Request { + var result []ctrl.Request + + operation, ok := o.Object.(*operatorv1.Operation) + if !ok { + return nil + } + + actual, err := listTaskGroupsByLabels(c, operation.Labels) + if err != nil { + return nil + } + + for _, ms := range actual.Items { + name := client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} + result = append(result, ctrl.Request{NamespacedName: name}) + } + + return result +} + +func getOwnerOperation(ctx context.Context, c client.Client, obj metav1.ObjectMeta) (*operatorv1.Operation, error) { + for _, ref := range obj.OwnerReferences { + if ref.Kind == "Operation" && ref.APIVersion == operatorv1.GroupVersion.String() { + operation := &operatorv1.Operation{} + key := client.ObjectKey{ + Namespace: obj.Namespace, + Name: ref.Name, + } + if err := c.Get(ctx, key, operation); err != nil { + return nil, errors.Wrapf(err, "error reading controller ref for %s/%s", obj.Namespace, obj.Name) + } + return operation, nil + } + } + return nil, errors.Errorf("missing controller ref for %s/%s", obj.Namespace, obj.Name) +} + +type matchingSelector struct { + selector labels.Selector +} + +func (m matchingSelector) ApplyToList(opts *client.ListOptions) { + opts.LabelSelector = m.selector +} + +func listNodesBySelector(c client.Client, selector *metav1.LabelSelector) (*corev1.NodeList, error) { + s, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, errors.Wrap(err, "failed to convert TaskGroup.Spec.NodeSelector to a selector") + } + + o := matchingSelector{selector: s} + + nodes := &corev1.NodeList{} + if err := c.List( + context.Background(), nodes, + o, + ); err != nil { + return nil, err + } + + return nodes, nil +} + +func filterNodes(nodes *corev1.NodeList, filter operatorv1.RuntimeTaskGroupNodeFilter) []corev1.Node { + if len(nodes.Items) == 0 { + return nodes.Items + } + + if filter == operatorv1.RuntimeTaskGroupNodeFilterAll || filter == operatorv1.RuntimeTaskGroupNodeUnknownFilter { + return nodes.Items + } + + // in order to ensure a predictable result, nodes are sorted by name before applying the filter + sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) + + if filter == operatorv1.RuntimeTaskGroupNodeFilterHead { + return nodes.Items[:1] + } + + // filter == operatorv1alpha1.TaskGroupNodeFilterTail + return nodes.Items[1:] +} + +func listTasksBySelector(c client.Client, selector *metav1.LabelSelector) (*operatorv1.RuntimeTaskList, error) { + selectorMap, err := metav1.LabelSelectorAsMap(selector) + if err != nil { + return nil, errors.Wrap(err, "failed to convert TaskGroup.Spec.Selector to a selector") + } + + tasks := &operatorv1.RuntimeTaskList{} + if err := c.List( + context.Background(), tasks, + client.MatchingLabels(selectorMap), + ); err != nil { + return nil, err + } + + return tasks, nil +} + +func taskGroupToTaskRequests(c client.Client, o handler.MapObject) []ctrl.Request { + var result []ctrl.Request + + taskgroup, ok := o.Object.(*operatorv1.RuntimeTaskGroup) + if !ok { + return nil + } + + actual, err := listTasksBySelector(c, &taskgroup.Spec.Selector) + if err != nil { + return nil + } + + for _, ms := range actual.Items { + name := client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} + result = append(result, ctrl.Request{NamespacedName: name}) + } + + return result +} + +func getOwnerTaskGroup(ctx context.Context, c client.Client, obj metav1.ObjectMeta) (*operatorv1.RuntimeTaskGroup, error) { + for _, ref := range obj.OwnerReferences { + if ref.Kind == "RuntimeTaskGroup" && ref.APIVersion == operatorv1.GroupVersion.String() { + taskgroup := &operatorv1.RuntimeTaskGroup{} + key := client.ObjectKey{ + Namespace: obj.Namespace, + Name: ref.Name, + } + if err := c.Get(ctx, key, taskgroup); err != nil { + return nil, errors.Wrapf(err, "error reading controller ref for %s/%s", obj.Namespace, obj.Name) + } + return taskgroup, nil + } + } + return nil, errors.Errorf("missing controller ref for %s/%s", obj.Namespace, obj.Name) +} diff --git a/controllers/util_test.go b/controllers/util_test.go new file mode 100644 index 0000000..a96b04d --- /dev/null +++ b/controllers/util_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/kubeadm/operator/api/v1alpha1" + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +//TODO: more tests + +func Test_filterNodes(t *testing.T) { + nodes := &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n3", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n2", + }, + }, + }, + } + + type args struct { + filter v1alpha1.RuntimeTaskGroupNodeFilter + } + tests := []struct { + name string + args args + want []corev1.Node + }{ + { + name: "filter all return all nodes", + args: args{ + filter: operatorv1.RuntimeTaskGroupNodeFilterAll, + }, + want: nodes.Items, + }, + { + name: "Filter head return the first node", + args: args{ + filter: operatorv1.RuntimeTaskGroupNodeFilterHead, + }, + want: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n1", + }, + }, + }, + }, + { + name: "Filter tail return the last two nodes", + args: args{ + filter: operatorv1.RuntimeTaskGroupNodeFilterTail, + }, + want: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n2", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "n3", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filterNodes(nodes, tt.args.filter); !reflect.DeepEqual(got, tt.want) { + t.Errorf("filterNodes() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/errors/operation.go b/errors/operation.go new file mode 100644 index 0000000..783d769 --- /dev/null +++ b/errors/operation.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package errors + +import ( + "fmt" +) + +// OperationStatusError defines error conditions for OperationStatus +type OperationStatusError string + +const ( + // OperationReplicaError represent error in creating RuntimeTaskGroups + OperationReplicaError OperationStatusError = "FailedReplics" + + // OperationReconciliationError represent an unexpected condition in controlled RuntimeTaskGroups + OperationReconciliationError OperationStatusError = "InvalidState" +) + +// OperationError provides a more descriptive kind of error that represents an error condition that +// should be set in the Operation.Status. The "Reason" field is meant for short, +// enum-style constants meant to be interpreted by machines. The "Message" +// field is meant to be read by humans. +type OperationError struct { + Reason OperationStatusError + Message string +} + +// The Error message +func (e *OperationError) Error() string { + return e.Message +} + +// NewOperationReplicaError returns a new OperationReplicaError +func NewOperationReplicaError(msg string, args ...interface{}) *OperationError { + return &OperationError{ + Reason: OperationReplicaError, + Message: fmt.Sprintf(msg, args...), + } +} + +// NewOperationReconciliationError returns a new OperationReconciliationError +func NewOperationReconciliationError(msg string, args ...interface{}) *OperationError { + return &OperationError{ + Reason: OperationReconciliationError, + Message: fmt.Sprintf(msg, args...), + } +} diff --git a/errors/runtimetask.go b/errors/runtimetask.go new file mode 100644 index 0000000..5c61843 --- /dev/null +++ b/errors/runtimetask.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package errors + +import ( + "fmt" +) + +// RuntimeTaskStatusError defines error conditions for RuntimeTaskStatus +type RuntimeTaskStatusError string + +const ( + // RuntimeTaskExecutionError represent an error during task execution + RuntimeTaskExecutionError RuntimeTaskStatusError = "ExecutionError" + + // RuntimeTaskIndexOutOfRangeError represent an error while accessing RuntimeTask commands + RuntimeTaskIndexOutOfRangeError RuntimeTaskStatusError = "IndexOutOfRange" +) + +// RuntimeTaskError provides a more descriptive kind of error that represents an error condition that +// should be set in the RuntimeTask.Status. The "Reason" field is meant for short, +// enum-style constants meant to be interpreted by machines. The "Message" +// field is meant to be read by humans. +type RuntimeTaskError struct { + Reason RuntimeTaskStatusError + Message string +} + +// The Error message +func (e *RuntimeTaskError) Error() string { + return e.Message +} + +// NewRuntimeTaskExecutionError returns a new RuntimeTaskExecutionError +func NewRuntimeTaskExecutionError(msg string, args ...interface{}) *RuntimeTaskError { + return &RuntimeTaskError{ + Reason: RuntimeTaskExecutionError, + Message: fmt.Sprintf(msg, args...), + } +} + +// NewRuntimeTaskIndexOutOfRangeError returns a new RuntimeTaskIndexOutOfRangeError +func NewRuntimeTaskIndexOutOfRangeError(msg string, args ...interface{}) *RuntimeTaskError { + return &RuntimeTaskError{ + Reason: RuntimeTaskIndexOutOfRangeError, + Message: fmt.Sprintf(msg, args...), + } +} diff --git a/errors/runtimetaskgroup.go b/errors/runtimetaskgroup.go new file mode 100644 index 0000000..101985a --- /dev/null +++ b/errors/runtimetaskgroup.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package errors + +import ( + "fmt" +) + +// RuntimeTaskGroupStatusError defines error conditions for RuntimeTaskGroupStatus +type RuntimeTaskGroupStatusError string + +const ( + // RuntimeTaskGroupReplicaError represent error in creating RuntimeTask replicas for a node + RuntimeTaskGroupReplicaError RuntimeTaskGroupStatusError = "FailedNodes" + + // RuntimeTaskGroupReconciliationError represent an unexpected condition in controlled RuntimeTasks + RuntimeTaskGroupReconciliationError RuntimeTaskGroupStatusError = "InvalidState" +) + +// RuntimeTaskGroupError provide a more descriptive kind of error that represents an error condition that +// should be set in the RuntimeTaskGroup.Status. The "Reason" field is meant for short, +// enum-style constants meant to be interpreted by machines. The "Message" +// field is meant to be read by humans. +type RuntimeTaskGroupError struct { + Reason RuntimeTaskGroupStatusError + Message string +} + +// The Error message +func (e *RuntimeTaskGroupError) Error() string { + return e.Message +} + +// NewRuntimeTaskGroupReplicaError returns a new RuntimeTaskGroupReplicaError +func NewRuntimeTaskGroupReplicaError(msg string, args ...interface{}) *RuntimeTaskGroupError { + return &RuntimeTaskGroupError{ + Reason: RuntimeTaskGroupReplicaError, + Message: fmt.Sprintf(msg, args...), + } +} + +// NewRuntimeTaskGroupReconciliationError returns a new RuntimeTaskGroupStatusError +func NewRuntimeTaskGroupReconciliationError(msg string, args ...interface{}) *RuntimeTaskGroupError { + return &RuntimeTaskGroupError{ + Reason: RuntimeTaskGroupReconciliationError, + Message: fmt.Sprintf(msg, args...), + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a9d6f7c --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module k8s.io/kubeadm/operator + +go 1.16 + +require ( + github.com/go-logr/logr v0.1.0 + github.com/pkg/errors v0.8.1 + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b + k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d + k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible + k8s.io/klog v0.3.1 + k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 + sigs.k8s.io/cluster-api v0.0.0-20190820130448-9fd8e4cbea0f + sigs.k8s.io/controller-runtime v0.2.2 + sigs.k8s.io/controller-tools v0.2.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cc78e76 --- /dev/null +++ b/go.sum @@ -0,0 +1,580 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest/autorest v0.2.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= +github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/flect v0.1.5 h1:xpKq9ap8MbYfhuPCF0dBH854Gp9CxZjr/IocxELFflo= +github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= +github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c h1:MUyE44mTvnI5A0xrxIxaMqoWFzPfQvtE2IWUollMDMs= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf h1:kt3wY1Lu5MJAnKTfoMR52Cu4gwvna4VTzNOiT8tY73s= +golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190501045030-23463209683d h1:D7DVZUZEUgsSIDTivnUtVeGfN5AvhDIKtdIZAqx0ieE= +golang.org/x/tools v0.0.0-20190501045030-23463209683d/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22 h1:0efs3hwEZhFKsCoP8l6dDB1AZWMgnEl3yWXWRZTOaEA= +gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b h1:aBGgKJUM9Hk/3AE8WaZIApnTxG35kbuQba2w+SXqezo= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8 h1:q1Qvjzs/iEdXF6A1a8H3AKVFDzJNcJn3nXMs6R6qFtA= +k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d h1:Jmdtdt1ZnoGfWWIIik61Z7nKYgO3J+swQJtPYsP9wHA= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apiserver v0.0.0-20190409021813-1ec86e4da56c/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w= +k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible h1:U5Bt+dab9K8qaUmXINrkXO135kA11/i5Kg1RUydgaMQ= +k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/component-base v0.0.0-20190409021516-bd2732e5c3f7/go.mod h1:DMaomcf3j3MM2j1FsvlLVVlc7wA2jPytEur3cP9zRxQ= +k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c h1:3KSCztE7gPitlZmWbNwue/2U0YruD65DqX3INopDAQM= +k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 h1:VBM/0P5TWxwk+Nw6Z+lAw3DKgO76g90ETOiA6rfLV1Y= +k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/cluster-api v0.0.0-20190820130448-9fd8e4cbea0f h1:gL8HQOSMGhMvL2BZNQSuKdXe02se34Uf29Rvq3iOfP8= +sigs.k8s.io/cluster-api v0.0.0-20190820130448-9fd8e4cbea0f/go.mod h1:zBOsRI6nVKzD6CPmA/3qYpLCckc5FVxMBxhvBxuDNsg= +sigs.k8s.io/controller-runtime v0.2.0-rc.0/go.mod h1:HweyYKQ8fBuzdu2bdaeBJvsFgAi/OqBBnrVGXcqKhME= +sigs.k8s.io/controller-runtime v0.2.2 h1:JT/vJJhUjjL9NZNwnm8AXmqCBUXSCFKmTaNjwDi28N0= +sigs.k8s.io/controller-runtime v0.2.2/go.mod h1:9dyohw3ZtoXQuV1e766PHUn+cmrRCIcBh6XIMFNMZ+I= +sigs.k8s.io/controller-tools v0.2.0-rc.0/go.mod h1:8t/X+FVWvk6TaBcsa+UKUBbn7GMtvyBKX30SGl4em6Y= +sigs.k8s.io/controller-tools v0.2.1 h1:HoCik83vXOpPi7KSJWdPRmiGntyOzK0v0BTV4U+pl8o= +sigs.k8s.io/controller-tools v0.2.1/go.mod h1:cenyhL7t2e7izk/Zy7ZxDqQ9YEj0niU5VDL1PWMgZ5s= +sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/testing_frameworks v0.1.2-0.20190130140139-57f07443c2d4 h1:GtDhkj3cF4A4IW+A9LScsuxvJqA9DE7G7PGH1f8B07U= +sigs.k8s.io/testing_frameworks v0.1.2-0.20190130140139-57f07443c2d4/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..d82aa77 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,14 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/fetch_bins.sh b/hack/fetch_bins.sh new file mode 100644 index 0000000..ccd5c91 --- /dev/null +++ b/hack/fetch_bins.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Enable tracing in this script off by setting the TRACE variable in your +# environment to any value: +# +# $ TRACE=1 test.sh +TRACE=${TRACE:-""} +if [[ -n "${TRACE}" ]]; then + set -x +fi + +k8s_version=1.14.1 +goarch=amd64 +goos="unknown" + +if [[ "${OSTYPE}" == "linux"* ]]; then + goos="linux" +elif [[ "${OSTYPE}" == "darwin"* ]]; then + goos="darwin" +fi + +if [[ "$goos" == "unknown" ]]; then + echo "OS '$OSTYPE' not supported. Aborting." >&2 + exit 1 +fi + +# Turn colors in this script off by setting the NO_COLOR variable in your +# environment to any value: +# +# $ NO_COLOR=1 test.sh +NO_COLOR=${NO_COLOR:-""} +if [[ -z "${NO_COLOR}" ]]; then + header=$'\e[1;33m' + reset=$'\e[0m' +else + header='' + reset='' +fi + +function header_text { + echo "$header$*$reset" +} + +tmp_root=/tmp + +kb_root_dir=${tmp_root}/kubebuilder + +# Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable +# in your environment to any value: +# +# $ SKIP_FETCH_TOOLS=1 ./fetch_ext_bins.sh +# +# If you skip fetching tools, this script will use the tools already on your +# machine, but rebuild the kubebuilder and kubebuilder-bin binaries. +SKIP_FETCH_TOOLS=${SKIP_FETCH_TOOLS:-""} + +function prepare_staging_dir { + header_text "preparing staging dir" + + if [[ -z "${SKIP_FETCH_TOOLS}" ]]; then + rm -rf "${kb_root_dir}" + else + rm -f "${kb_root_dir}/kubebuilder/bin/kubebuilder" + rm -f "${kb_root_dir}/kubebuilder/bin/kubebuilder-gen" + rm -f "${kb_root_dir}/kubebuilder/bin/vendor.tar.gz" + fi +} + +# fetch k8s API gen tools and make it available under kb_root_dir/bin. +function fetch_tools { + if [[ -n "$SKIP_FETCH_TOOLS" ]]; then + return 0 + fi + + header_text "fetching tools" + kb_tools_archive_name="kubebuilder-tools-${k8s_version}-${goos}-${goarch}.tar.gz" + kb_tools_download_url="https://storage.googleapis.com/kubebuilder-tools/${kb_tools_archive_name}" + + kb_tools_archive_path="${tmp_root}/${kb_tools_archive_name}" + if [[ ! -f ${kb_tools_archive_path} ]]; then + curl -fsL ${kb_tools_download_url} -o "${kb_tools_archive_path}" + fi + tar -zvxf "${kb_tools_archive_path}" -C "${tmp_root}/" +} + +function setup_envs { + header_text "setting up env vars" + + # Setup env vars + export PATH=/tmp/kubebuilder/bin:$PATH + export TEST_ASSET_KUBECTL=/tmp/kubebuilder/bin/kubectl + export TEST_ASSET_KUBE_APISERVER=/tmp/kubebuilder/bin/kube-apiserver + export TEST_ASSET_ETCD=/tmp/kubebuilder/bin/etcd +} diff --git a/hack/set-workspace-status.sh b/hack/set-workspace-status.sh new file mode 100755 index 0000000..a4d06b7 --- /dev/null +++ b/hack/set-workspace-status.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Override by passing in VERSION +VERSION=${VERSION:-$(git tag --sort taggerdate | tail -n 1)} + +# Override by passing in SHORT +SHORT=${SHORT:-$(git rev-parse --short "${VERSION}")} diff --git a/hack/tools/go.mod b/hack/tools/go.mod new file mode 100644 index 0000000..0571804 --- /dev/null +++ b/hack/tools/go.mod @@ -0,0 +1,5 @@ +module sigs.k8s.io/cluster-api-provider-docker/hack/tools + +go 1.16 + +require sigs.k8s.io/controller-tools v0.2.0-rc.0 diff --git a/hack/tools/go.sum b/hack/tools/go.sum new file mode 100644 index 0000000..f739d96 --- /dev/null +++ b/hack/tools/go.sum @@ -0,0 +1,96 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gobuffalo/flect v0.1.5 h1:xpKq9ap8MbYfhuPCF0dBH854Gp9CxZjr/IocxELFflo= +github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190501045030-23463209683d h1:D7DVZUZEUgsSIDTivnUtVeGfN5AvhDIKtdIZAqx0ieE= +golang.org/x/tools v0.0.0-20190501045030-23463209683d/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b h1:aBGgKJUM9Hk/3AE8WaZIApnTxG35kbuQba2w+SXqezo= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8 h1:q1Qvjzs/iEdXF6A1a8H3AKVFDzJNcJn3nXMs6R6qFtA= +k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d h1:Jmdtdt1ZnoGfWWIIik61Z7nKYgO3J+swQJtPYsP9wHA= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c= +k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +sigs.k8s.io/controller-tools v0.2.0-rc.0 h1:8FZR8qgxNPPBCb6Q/WwoRUfYqWvgn1Fz6m5uKcCbXfI= +sigs.k8s.io/controller-tools v0.2.0-rc.0/go.mod h1:8t/X+FVWvk6TaBcsa+UKUBbn7GMtvyBKX30SGl4em6Y= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/tools/tools.go b/hack/tools/tools.go new file mode 100644 index 0000000..d442b28 --- /dev/null +++ b/hack/tools/tools.go @@ -0,0 +1,24 @@ +// +build tools + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package tools imports things required by build scripts, to force `go mod` to see them as dependencies +package tools + +import ( + _ "sigs.k8s.io/controller-tools/cmd/controller-gen" // nolint +) diff --git a/hack/update-all.sh b/hack/update-all.sh new file mode 100755 index 0000000..f0c5071 --- /dev/null +++ b/hack/update-all.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +REPO_PATH=$(get_root_path) + +"${REPO_PATH}"/hack/update-deps.sh +"${REPO_PATH}"/hack/update-gofmt.sh +"${REPO_PATH}"/hack/update-goimports.sh diff --git a/hack/update-deps.sh b/hack/update-deps.sh new file mode 100755 index 0000000..7590706 --- /dev/null +++ b/hack/update-deps.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o nounset +set -o errexit +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +GOPROXY=$(go env GOPROXY) +export GOPROXY="${GOPROXY:-https://proxy.golang.org}" +export GO111MODULE="on" +go mod tidy diff --git a/hack/update-gofmt.sh b/hack/update-gofmt.sh new file mode 100755 index 0000000..ed120eb --- /dev/null +++ b/hack/update-gofmt.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# script to run gofmt over our code (not vendor) +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# update go fmt +go fmt ./... diff --git a/hack/update-goimports.sh b/hack/update-goimports.sh new file mode 100755 index 0000000..9cb5949 --- /dev/null +++ b/hack/update-goimports.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# script to run gofmt over our code (not vendor) +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# update go imports, skipping generated files +git ls-files | grep "\.go$" | grep -v -e "zz_generated" | xargs goimports -local k8s.io/kubeadm/operator -w diff --git a/hack/utils.sh b/hack/utils.sh new file mode 100644 index 0000000..1f90f18 --- /dev/null +++ b/hack/utils.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# get_root_path returns the root path of the project source tree +get_root_path() { + # shellcheck disable=SC2005 + echo "$(git rev-parse --show-toplevel)/operator" +} + +# cd_root_path cds to the root path of the project source tree +cd_root_path() { + cd "$(get_root_path)" || exit +} diff --git a/hack/verify-all.sh b/hack/verify-all.sh new file mode 100755 index 0000000..2f12f05 --- /dev/null +++ b/hack/verify-all.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" + +# cd to the root path +cd_root_path + +failure() { + if [[ "${1}" != 0 ]]; then + res=1 + failed+=("${2}") + outputs+=("${3}") + fi +} + +# exit code, if a script fails we'll set this to 1 +res=0 +failed=() +outputs=() + +# run all verify scripts, optionally skipping any of them + +if [[ "${VERIFY_WHITESPACE:-true}" == "true" ]]; then + echo "[*] Verifying whitespace..." + out=$(hack/verify-whitespace.sh 2>&1) + failure $? "verify-whitespace.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_SPELLING:-true}" == "true" ]]; then + echo "[*] Verifying spelling..." + out=$(hack/verify-spelling.sh 2>&1) + failure $? "verify-spelling.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_BOILERPLATE:-true}" == "true" ]]; then + echo "[*] Verifying boilerplate..." + out=$(hack/verify-boilerplate.sh 2>&1) + failure $? "verify-boilerplate.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_GOFMT:-true}" == "true" ]]; then + echo "[*] Verifying gofmt..." + out=$(hack/verify-gofmt.sh 2>&1) + failure $? "verify-gofmt.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_GOIMPORTS:-true}" == "true" ]]; then + echo "[*] Verifying goimports..." + out=$(hack/verify-goimports.sh 2>&1) + failure $? "verify-goimports.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_GOLINT:-true}" == "true" ]]; then + echo "[*] Verifying golint..." + out=$(hack/verify-golint.sh 2>&1) + failure $? "verify-golint.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_GOVET:-true}" == "true" ]]; then + echo "[*] Verifying govet..." + out=$(hack/verify-govet.sh 2>&1) + failure $? "verify-govet.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_DEPS:-true}" == "true" ]]; then + echo "[*] Verifying deps..." + out=$(hack/verify-deps.sh 2>&1) + failure $? "verify-deps.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_GOTEST:-true}" == "true" ]]; then + echo "[*] Verifying gotest..." + out=$(hack/verify-gotest.sh 2>&1) + failure $? "verify-gotest.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_BUILD:-true}" == "true" ]]; then + echo "[*] Verifying build..." + out=$(hack/verify-build.sh 2>&1) + failure $? "verify-build.sh" "${out}" + cd_root_path +fi + +if [[ "${VERIFY_DOCKER_BUILD:-true}" == "true" ]]; then + echo "[*] Verifying manager docker image build..." + out=$(hack/verify-docker-build.sh 2>&1) + failure $? "verify-docker-build.sh" "${out}" + cd_root_path +fi + +# exit based on verify scripts +if [[ "${res}" = 0 ]]; then + echo "" + echo "All verify checks passed, congrats!" +else + echo "" + echo "Some of the verify scripts failed:" + for i in "${!failed[@]}"; do + echo "- ${failed[$i]}:" + echo "${outputs[$i]}" + echo + done +fi +exit "${res}" diff --git a/hack/verify-boilerplate.go b/hack/verify-boilerplate.go new file mode 100644 index 0000000..a49cd59 --- /dev/null +++ b/hack/verify-boilerplate.go @@ -0,0 +1,174 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" +) + +const ( + yearPlaceholder = "YEAR" + boilerPlateStart = "Copyright " + boilerPlateEnd = "limitations under the License." +) + +var ( + supportedExt = []string{".go", ".py", ".sh"} + yearRegexp = regexp.MustCompile("(20)[0-9][0-9]") + boilerPlate = []string{ + boilerPlateStart + yearPlaceholder + " The Kubernetes Authors.", + "", + `Licensed under the Apache License, Version 2.0 (the "License");`, + "you may not use this file except in compliance with the License.", + "You may obtain a copy of the License at", + "", + " http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + `distributed under the License is distributed on an "AS IS" BASIS,`, + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + boilerPlateEnd, + } +) + +// trimLeadingComment strips a single line comment characters such as # or // +// at the exact beginning of a line, but also the first possible space character after it. +func trimLeadingComment(line, c string) string { + if strings.Index(line, c) == 0 { + x := len(c) + if len(line) == x { + return "" + } + if line[x] == byte(' ') { + return line[x+1:] + } + return line[x:] + } + return line +} + +// verifyFileExtension verifies if the file extensions is supported +func isSupportedFileExtension(filePath string) bool { + // check if the file has an extension + idx := strings.LastIndex(filePath, ".") + if idx == -1 { + return false + } + + // check if the file has a supported extension + ext := filePath[idx : idx+len(filePath)-idx] + for _, e := range supportedExt { + if e == ext { + return true + } + } + return false +} + +// verifyBoilerplate verifies if a string contains the boilerplate +func verifyBoilerplate(contents string) error { + idx := 0 + foundBoilerplateStart := false + lines := strings.Split(contents, "\n") + for _, line := range lines { + // handle leading comments + line = trimLeadingComment(line, "//") + line = trimLeadingComment(line, "#") + + // find the start of the boilerplate + bpLine := boilerPlate[idx] + if strings.Contains(line, boilerPlateStart) { + foundBoilerplateStart = true + + // validate the year of the copyright + yearWords := strings.Split(line, " ") + expectedLen := len(strings.Split(boilerPlate[0], " ")) + if len(yearWords) != expectedLen { + return fmt.Errorf("copyright line should contain exactly %d words", expectedLen) + } + if !yearRegexp.MatchString(yearWords[1]) { + return fmt.Errorf("cannot parse the year in the copyright line") + } + bpLine = strings.ReplaceAll(bpLine, yearPlaceholder, yearWords[1]) + } + + // match line by line + if foundBoilerplateStart { + if line != bpLine { + return fmt.Errorf("boilerplate line %d does not match\nexpected: %q\ngot: %q", idx+1, bpLine, line) + } + idx++ + // exit after the last line is found + if strings.Index(line, boilerPlateEnd) == 0 { + break + } + } + } + + if !foundBoilerplateStart { + return fmt.Errorf("the file is missing a boilerplate") + } + if idx < len(boilerPlate) { + return errors.New("boilerplate has missing lines") + } + return nil +} + +// verifyFile verifies if a file contains the boilerplate +func verifyFile(filePath string) error { + if len(filePath) == 0 { + return errors.New("empty file name") + } + + if !isSupportedFileExtension(filePath) { + fmt.Printf("skipping %q: unsupported file type\n", filePath) + return nil + } + + // read the file + b, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + + return verifyBoilerplate(string(b)) +} + +func main() { + if len(os.Args) < 2 { + fmt.Println("usage: " + + "go run verify-boilerplate.go ...") + os.Exit(1) + } + + hasErr := false + for _, filePath := range os.Args[1:] { + if err := verifyFile(filePath); err != nil { + fmt.Printf("error validating %q: %v\n", filePath, err) + hasErr = true + } + } + if hasErr { + os.Exit(1) + } +} diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh new file mode 100755 index 0000000..c8b7890 --- /dev/null +++ b/hack/verify-boilerplate.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +git ls-files | grep --invert-match "\.deepcopy\.go" | grep --invert-match "^third_party" | xargs go run ./hack/verify-boilerplate.go diff --git a/hack/verify-boilerplate_test.go b/hack/verify-boilerplate_test.go new file mode 100644 index 0000000..c87b21c --- /dev/null +++ b/hack/verify-boilerplate_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestVerifyBoilerPlate(t *testing.T) { + testcases := []struct { + name string + bp string + expectedError bool + }{ + { + name: "valid: boilerplate is valid", + bp: `\/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */`, + expectedError: false, + }, + { + name: "invalid: missing lines", + bp: ` +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +`, + expectedError: true, + }, + { + name: "invalid: bad year", + bp: "Copyright 1019 The Kubernetes Authors.", + expectedError: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + if err := verifyBoilerplate(tc.bp); err != nil != tc.expectedError { + t.Errorf("expected error: %v, got: %v, error: %v", tc.expectedError, err != nil, err) + } + }) + } +} + +func TestTrimLeadingComment(t *testing.T) { + testcases := []struct { + name string + comment string + line string + expectedResult string + }{ + { + name: "trim leading comment", + comment: "#", + line: "# test", + expectedResult: "test", + }, + { + name: "empty line", + comment: "#", + line: "#", + expectedResult: "", + }, + { + name: "trim leading comment and space", + comment: "//", + line: "// test", + expectedResult: "test", + }, + { + name: "no comment", + comment: "//", + line: "test", + expectedResult: "test", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + if res := trimLeadingComment(tc.line, tc.comment); res != tc.expectedResult { + t.Errorf("expected: %q, got: %q", tc.expectedResult, res) + } + }) + } +} diff --git a/hack/verify-build.sh b/hack/verify-build.sh new file mode 100755 index 0000000..e8f4a1b --- /dev/null +++ b/hack/verify-build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" + +# check if the code builds +cd_root_path +export GO111MODULE=on +go build -o bin/manager main.go diff --git a/hack/verify-deps.sh b/hack/verify-deps.sh new file mode 100755 index 0000000..cacc490 --- /dev/null +++ b/hack/verify-deps.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# cleanup on exit +cleanup() { + echo "Cleaning up..." + mv go.mod.old go.mod + mv go.sum.old go.sum +} +trap cleanup EXIT + +echo "Verifying..." +# temporary copy the go mod and sum files +cp go.mod go.mod.old || exit +cp go.sum go.sum.old || exit + +# run update-deps.sh +export GO111MODULE="on" +./hack/update-deps.sh + +# compare the old and new files +DIFF0=$(diff -u go.mod go.mod.old) +DIFF1=$(diff -u go.sum go.sum.old) + +if [[ -n "${DIFF0}" ]] || [[ -n "${DIFF1}" ]]; then + echo "${DIFF0}" + echo "${DIFF1}" + echo "Check failed. Please run ./hack/update-deps.sh" + exit 1 +fi diff --git a/hack/verify-docker-build.sh b/hack/verify-docker-build.sh new file mode 100755 index 0000000..baeb71b --- /dev/null +++ b/hack/verify-docker-build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" + +# check if manager docker image builds +cd_root_path + +export GO111MODULE=on +go mod download +go build -o bin/manager main.go +docker build --file Dockerfile -t manager:pr-verify . diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh new file mode 100755 index 0000000..7731f11 --- /dev/null +++ b/hack/verify-gofmt.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# check for gofmt diffs +diff=$(git ls-files | grep "\.go$" | grep -v "\/vendor" | xargs gofmt -s -d 2>&1) +if [[ -n "${diff}" ]]; then + echo "${diff}" + echo + echo "Check failed. Please run hack/update-gofmt.sh" + exit 1 +fi diff --git a/hack/verify-goimports.sh b/hack/verify-goimports.sh new file mode 100755 index 0000000..3b748a7 --- /dev/null +++ b/hack/verify-goimports.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup +exitHandler() ( + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +) +trap exitHandler EXIT + +# pull goimports +export GO111MODULE=on +URL="https://github.com/golang/tools.git" +git clone --quiet --depth=1 "${URL}" "${TMP_DIR}" +pushd "${TMP_DIR}" > /dev/null +popd > /dev/null + +# build goimports +BIN_PATH="${TMP_DIR}/cmd/goimports" +pushd "${BIN_PATH}" > /dev/null +echo "Building goimports..." +go build > /dev/null +popd > /dev/null + +# check for goimports diffs +diff=$(git ls-files | grep "\.go$" | grep -v -e "zz_generated" | xargs "${BIN_PATH}/goimports" -local k8s.io/kubeadm/operator -d 2>&1) +if [[ -n "${diff}" ]]; then + echo "${diff}" + echo + echo "Check failed. Please run hack/update-goimports.sh" + exit 1 +fi diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh new file mode 100755 index 0000000..75dce90 --- /dev/null +++ b/hack/verify-golint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CI script to run go lint over our code +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" + +# cd to the root path +REPO_PATH=$(get_root_path) + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup +exitHandler() ( + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +) +trap exitHandler EXIT + +# pull the source code and build the binary +cd "${TMP_DIR}" +URL="https://github.com/golang/lint" +echo "Cloning ${URL} in ${TMP_DIR}..." +git clone --quiet --depth=1 "${URL}" . +echo "Building golint..." +export GO111MODULE=on +go build -o ./golint/golint ./golint + +# run the binary +cd "${REPO_PATH}" +echo "Running golint..." +git ls-files | grep "\.go$" | \ + grep -v "\\/vendor\\/" | \ + xargs -L1 "${TMP_DIR}/golint/golint" -set_exit_status diff --git a/hack/verify-gotest.sh b/hack/verify-gotest.sh new file mode 100755 index 0000000..7f4a2ce --- /dev/null +++ b/hack/verify-gotest.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# install kubebuilder tools for tests +# shellcheck disable=SC1090 +source "$(dirname "$0")/fetch_bins.sh" +fetch_tools + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# run go test +export GO111MODULE=on +setup_envs && go test ./... diff --git a/hack/verify-govet.sh b/hack/verify-govet.sh new file mode 100755 index 0000000..ccfe41f --- /dev/null +++ b/hack/verify-govet.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CI script to run go vet over our code +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# run go vet +export GO111MODULE=on +go vet ./... diff --git a/hack/verify-shellcheck.sh b/hack/verify-shellcheck.sh new file mode 100755 index 0000000..e0a1785 --- /dev/null +++ b/hack/verify-shellcheck.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +ROOT_PATH=$(get_root_path) +os=$(go env GOOS) + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup on exit +cleanup() { + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +# install shellcheck +cd "${TMP_DIR}" || exit +VERSION="shellcheck-stable" +DOWNLOAD_FILE="${VERSION}.${os}.x86_64.tar.xz" +wget https://github.com/koalaman/shellcheck/releases/download/stable/"${DOWNLOAD_FILE}" +tar xf "${DOWNLOAD_FILE}" +cd "${VERSION}" || exit + +echo "Running shellcheck..." +cd "${ROOT_PATH}" || exit +OUT="${TMP_DIR}/out.log" +FILES=$(find . -name "*.sh") +while read -r file; do + "${TMP_DIR}/${VERSION}/shellcheck" "$file" >> "${OUT}" 2>&1 +done <<< "$FILES" + +if [[ -s "${OUT}" ]]; then + echo "Found errors:" + cat "${OUT}" + exit 1 +fi diff --git a/hack/verify-spelling.sh b/hack/verify-spelling.sh new file mode 100755 index 0000000..3883785 --- /dev/null +++ b/hack/verify-spelling.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup +exitHandler() ( + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +) +trap exitHandler EXIT + +# pull misspell +export GO111MODULE=on +URL="https://github.com/client9/misspell" +echo "Cloning ${URL} in ${TMP_DIR}..." +git clone --quiet --depth=1 "${URL}" "${TMP_DIR}" +pushd "${TMP_DIR}" > /dev/null +go mod init misspell +popd > /dev/null + +# build misspell +BIN_PATH="${TMP_DIR}/cmd/misspell" +pushd "${BIN_PATH}" > /dev/null +echo "Building misspell..." +go build > /dev/null +popd > /dev/null + +# check spelling +RES=0 +ERROR_LOG="${TMP_DIR}/errors.log" +echo "Checking spelling..." +git ls-files | grep -v -e vendor | xargs "${BIN_PATH}/misspell" > "${ERROR_LOG}" +if [[ -s "${ERROR_LOG}" ]]; then + sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status + echo "Found spelling errors!" + RES=1 +fi +exit "${RES}" diff --git a/hack/verify-whitespace.sh b/hack/verify-whitespace.sh new file mode 100755 index 0000000..c277a38 --- /dev/null +++ b/hack/verify-whitespace.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "$(dirname "$0")/utils.sh" +# cd to the root path +cd_root_path + +echo "Verifying trailing whitespace..." +TRAILING="$(grep -rnI '[[:blank:]]$' . | grep -v -e .git)" + +ERR="0" +if [[ -n "$TRAILING" ]]; then + echo "Found trailing whitespace in the follow files:" + echo "${TRAILING}" + ERR="1" +fi + +echo -e "Verifying new lines at end of files..." +FILES="$(git ls-files | grep -I -v -e vendor)" +while read -r LINE; do + grep -qI . "${LINE}" || continue # skip binary files + c="$(tail -c 1 "${LINE}")" + if [[ "$c" != "" ]]; then + echo "${LINE}: no newline at the end of file" + ERR=1 + fi +done <<< "${FILES}" + +if [[ "$ERR" == "1" ]]; then + echo "Found whitespace errors!" + exit 1 +fi diff --git a/main.go b/main.go new file mode 100644 index 0000000..749db6e --- /dev/null +++ b/main.go @@ -0,0 +1,158 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/klog" + "k8s.io/klog/klogr" + ctrl "sigs.k8s.io/controller-runtime" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" + "k8s.io/kubeadm/operator/controllers" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + + _ = operatorv1.AddToScheme(scheme) + // +kubebuilder:scaffold:scheme +} + +type managerMode string + +const ( + modeManager = managerMode("manager") + modeAgent = managerMode("agent") +) + +func main() { + klog.InitFlags(nil) + var mode string + var pod string + var namespace string + var image string + var nodeName string + var operation string + var metricsAddr string + var metricsRBAC bool + var enableLeaderElection bool + + // common flags + flag.StringVar(&mode, "mode", string(modeManager), "One of [manger, agent]") + flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to") + + // manager flags + flag.StringVar(&pod, "manager-pod", "", "The pod the manager is running in") + flag.StringVar(&namespace, "manager-namespace", "", "The namespace the manager is running in") //TODO: implement in all the controllers + flag.StringVar(&image, "agent-image", "", "The image that should be used for agent the DaemonSet. If empty, the manager image will be used") //TODO: remove; always use manager image + flag.BoolVar(&metricsRBAC, "agent-metrics-rbac", true, "Use RBAC authn/z for the /metrics endpoint of agents") + flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager") + + // agent flags + flag.StringVar(&nodeName, "agent-node-name", "", "The node that the agent manager should control") + flag.StringVar(&operation, "agent-operation", "", "The operation that the agent manager should control. If empty, the agent will control headless Task only") + + flag.Parse() + + ctrl.SetLogger(klogr.New()) + + if managerMode(mode) != modeManager && managerMode(mode) != modeAgent { + setupLog.Error(errors.New("invalid value"), "unable to create controllers with an invalid --mode value") + os.Exit(1) + } + + if managerMode(mode) == modeAgent { + enableLeaderElection = false + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if managerMode(mode) == modeManager { + if err = (&controllers.OperationReconciler{ + Client: mgr.GetClient(), + ManagerContainerName: pod, + ManagerNamespace: namespace, + AgentImage: image, + MetricsRBAC: metricsRBAC, + Log: ctrl.Log.WithName("controllers").WithName("Operation"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Operation") + os.Exit(1) + } + + if err = (&controllers.RuntimeTaskGroupReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RuntimeTaskGroup"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RuntimeTaskGroup") + os.Exit(1) + } + + setupLog.Info("starting controller manager", "manager-pod", pod, "manager-namespace", namespace, "agent-image", image, "agent-metrics-RBAC", metricsRBAC) + } + + if managerMode(mode) == modeAgent { + if nodeName == "" { + setupLog.Error(err, "unable to create controller without the --agent-node-name value set", "controller", "RuntimeTask") + os.Exit(1) + } + if nodeName == "" { + setupLog.Error(err, "unable to create controller without the --agent-operation value set", "controller", "RuntimeTask") + os.Exit(1) + } + + if err = (&controllers.RuntimeTaskReconciler{ + Client: mgr.GetClient(), + NodeName: nodeName, + Operation: operation, + Log: ctrl.Log.WithName("controllers").WithName("RuntimeTask").WithValues("node-name", nodeName), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RuntimeTask") + os.Exit(1) + } + setupLog.Info("starting agent manager", "agent-node", nodeName, "agent-operation", operation) + } + + // +kubebuilder:scaffold:builder + + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/operations/custom.go b/operations/custom.go new file mode 100644 index 0000000..70b9573 --- /dev/null +++ b/operations/custom.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + "fmt" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func setupCustom() map[string]string { + return map[string]string{} +} + +func planCustom(operation *operatorv1.Operation, spec *operatorv1.CustomOperationSpec) *operatorv1.RuntimeTaskGroupList { + var items []operatorv1.RuntimeTaskGroup + + for i, t := range spec.Workflow { + items = append(items, fixupCustomTaskGroup(operation, t, fmt.Sprintf("%02d", i+1))) + } + + return &operatorv1.RuntimeTaskGroupList{ + Items: items, + } +} diff --git a/operations/factory.go b/operations/factory.go new file mode 100644 index 0000000..5d09e95 --- /dev/null +++ b/operations/factory.go @@ -0,0 +1,57 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + "github.com/pkg/errors" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +// DaemonSetNodeSelectorLabels labels for limiting the nodes where the operation agent will be deployed +func DaemonSetNodeSelectorLabels(operation *operatorv1.Operation) (map[string]string, error) { + if operation.Spec.RenewCertificates != nil { + return setupRenewCertificates(), nil + } + + if operation.Spec.Upgrade != nil { + return setupUpgrade(), nil + } + + if operation.Spec.CustomOperation != nil { + return setupCustom(), nil + } + + return nil, errors.New("Invalid Operation.Spec.OperatorDescriptor. There are no operation implementation matching this spec") +} + +// TaskGroupList return the list of TaskGroup to be performed by an operation +func TaskGroupList(operation *operatorv1.Operation) (*operatorv1.RuntimeTaskGroupList, error) { + if operation.Spec.RenewCertificates != nil { + return planRenewCertificates(operation, operation.Spec.RenewCertificates), nil + } + + if operation.Spec.Upgrade != nil { + return planUpgrade(operation, operation.Spec.Upgrade), nil + } + + if operation.Spec.CustomOperation != nil { + return planCustom(operation, operation.Spec.CustomOperation), nil + } + + return nil, errors.New("Invalid Operation.Spec.OperatorDescriptor. There are no operation implementation matching this spec") +} diff --git a/operations/renewcertificates.go b/operations/renewcertificates.go new file mode 100644 index 0000000..568a61a --- /dev/null +++ b/operations/renewcertificates.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func setupRenewCertificates() map[string]string { + return map[string]string{ + "node-role.kubernetes.io/master": "", + } +} + +func planRenewCertificates(operation *operatorv1.Operation, spec *operatorv1.RenewCertificatesOperationSpec) *operatorv1.RuntimeTaskGroupList { + var items []operatorv1.RuntimeTaskGroup + + t1 := createBasicTaskGroup(operation, "01", "renew-cp") + setCPSelector(&t1) + t1.Spec.Template.Spec.Commands = append(t1.Spec.Template.Spec.Commands, + operatorv1.CommandDescriptor{ + KubeadmRenewCertificates: &operatorv1.KubeadmRenewCertsCommandSpec{}, + }, + ) + items = append(items, t1) + + return &operatorv1.RuntimeTaskGroupList{ + Items: items, + } +} diff --git a/operations/upgrade.go b/operations/upgrade.go new file mode 100644 index 0000000..e606684 --- /dev/null +++ b/operations/upgrade.go @@ -0,0 +1,85 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func setupUpgrade() map[string]string { + return map[string]string{} +} + +func planUpgrade(operation *operatorv1.Operation, spec *operatorv1.UpgradeOperationSpec) *operatorv1.RuntimeTaskGroupList { + var items []operatorv1.RuntimeTaskGroup + + t1 := createBasicTaskGroup(operation, "01", "upgrade-cp-1") + setCP1Selector(&t1) + t1.Spec.NodeFilter = string(operatorv1.RuntimeTaskGroupNodeFilterHead) + t1.Spec.Template.Spec.Commands = append(t1.Spec.Template.Spec.Commands, + operatorv1.CommandDescriptor{ + UpgradeKubeadm: &operatorv1.UpgradeKubeadmCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + KubeadmUpgradeApply: &operatorv1.KubeadmUpgradeApplyCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + UpgradeKubeletAndKubeactl: &operatorv1.UpgradeKubeletAndKubeactlCommandSpec{}, + }, + ) + items = append(items, t1) + + t2 := createBasicTaskGroup(operation, "02", "upgrade-cp-n") + setCPNSelector(&t2) + t2.Spec.Template.Spec.Commands = append(t2.Spec.Template.Spec.Commands, + operatorv1.CommandDescriptor{ + UpgradeKubeadm: &operatorv1.UpgradeKubeadmCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + KubeadmUpgradeNode: &operatorv1.KubeadmUpgradeNodeCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + UpgradeKubeletAndKubeactl: &operatorv1.UpgradeKubeletAndKubeactlCommandSpec{}, + }, + ) + items = append(items, t2) + + t3 := createBasicTaskGroup(operation, "02", "upgrade-w") + setWSelector(&t3) + t3.Spec.Template.Spec.Commands = append(t3.Spec.Template.Spec.Commands, + operatorv1.CommandDescriptor{ + KubectlDrain: &operatorv1.KubectlDrainCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + UpgradeKubeadm: &operatorv1.UpgradeKubeadmCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + KubeadmUpgradeNode: &operatorv1.KubeadmUpgradeNodeCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + UpgradeKubeletAndKubeactl: &operatorv1.UpgradeKubeletAndKubeactlCommandSpec{}, + }, + operatorv1.CommandDescriptor{ + KubectlUncordon: &operatorv1.KubectlUncordonCommandSpec{}, + }, + ) + items = append(items, t3) + + return &operatorv1.RuntimeTaskGroupList{ + Items: items, + } +} diff --git a/operations/util.go b/operations/util.go new file mode 100644 index 0000000..c533dea --- /dev/null +++ b/operations/util.go @@ -0,0 +1,127 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1" +) + +func createBasicTaskGroup(operation *operatorv1.Operation, taskdeploymentOrder string, taskdeploymentName string) operatorv1.RuntimeTaskGroup { + gv := operatorv1.GroupVersion + + labels := map[string]string{} + for k, v := range operation.Labels { + labels[k] = v + } + labels[operatorv1.TaskGroupNameLabel] = taskdeploymentName + labels[operatorv1.TaskGroupOrderLabel] = taskdeploymentOrder + + return operatorv1.RuntimeTaskGroup{ + TypeMeta: metav1.TypeMeta{ + Kind: gv.WithKind("TaskGroup").Kind, + APIVersion: gv.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s", operation.Name, taskdeploymentOrder, taskdeploymentName), //TODO: GeneratedName? + Labels: labels, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(operation, operation.GroupVersionKind())}, + }, + Spec: operatorv1.RuntimeTaskGroupSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: operatorv1.RuntimeTaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + CreationTimestamp: metav1.Now(), + }, + Spec: operatorv1.RuntimeTaskSpec{ + Commands: []operatorv1.CommandDescriptor{}, + }, + }, + }, + Status: operatorv1.RuntimeTaskGroupStatus{ + Phase: string(operatorv1.OperationPhasePending), + }, + } +} + +func setCPSelector(t *operatorv1.RuntimeTaskGroup) { + t.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + } +} + +func setCP1Selector(t *operatorv1.RuntimeTaskGroup) { + t.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + } + t.Spec.NodeFilter = string(operatorv1.RuntimeTaskGroupNodeFilterHead) +} + +func setCPNSelector(t *operatorv1.RuntimeTaskGroup) { + t.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + } + t.Spec.NodeFilter = string(operatorv1.RuntimeTaskGroupNodeFilterTail) +} + +func setWSelector(t *operatorv1.RuntimeTaskGroup) { + t.Spec.NodeSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "node-role.kubernetes.io/master", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + } +} + +func fixupCustomTaskGroup(operation *operatorv1.Operation, taskgroup operatorv1.RuntimeTaskGroup, taskdeploymentOrder string) operatorv1.RuntimeTaskGroup { + gv := operatorv1.GroupVersion + + //TODO: consider if to preserve labels from custom taskgroup, taskgroup.Spec.Selector, taskgroup.Spec.Template + + labels := map[string]string{} + for k, v := range operation.Labels { + labels[k] = v + } + labels[operatorv1.TaskGroupNameLabel] = taskgroup.Name + labels[operatorv1.TaskGroupOrderLabel] = taskdeploymentOrder + + taskgroup.SetGroupVersionKind(gv.WithKind("TaskGroup")) + taskgroup.SetLabels(labels) + taskgroup.SetOwnerReferences([]metav1.OwnerReference{*metav1.NewControllerRef(operation, operation.GroupVersionKind())}) + taskgroup.Spec.Selector.MatchLabels = labels + taskgroup.Spec.Template.GetObjectMeta().SetLabels(labels) + taskgroup.Spec.Template.SetCreationTimestamp(metav1.Now()) + taskgroup.Status = operatorv1.RuntimeTaskGroupStatus{ + Phase: string(operatorv1.OperationPhasePending), + } + + return taskgroup +}