kubeadm定制功能移植 about sealos HOT 2 CLOSED

From 5f5cb48c2316fd0382e69a9c31ba18067407d68a Mon Sep 17 00:00:00 2001
From: fanux <[email protected]>
Date: Sat, 22 Jun 2019 16:33:00 +0800
Subject: [PATCH] ass

add v1.15.1

add 1.14.4

update

v1.15.2-lvscare

add maxage to 99 years
---
 .bash_history                                 |  6 ++
 .drone.yml                                    | 27 +++++
 Note.md                                       | 86 ++++++++++++++++
 cmd/kubeadm/app/cmd/join.go                   | 24 ++++-
 cmd/kubeadm/app/localLB/local_lb.go           | 98 +++++++++++++++++++
 cmd/kubeadm/app/util/pkiutil/pki_helpers.go   |  3 +-
 cmd/kubeadm/deletetag.sh                      |  2 +
 cmd/kubeadm/kubeadm.go                        |  4 +
 go.mod                                        |  1 +
 .../src/k8s.io/client-go/util/cert/cert.go    |  6 +-
 10 files changed, 252 insertions(+), 5 deletions(-)
 create mode 100644 .bash_history
 create mode 100644 .drone.yml
 create mode 100644 Note.md
 create mode 100644 cmd/kubeadm/app/localLB/local_lb.go
 create mode 100644 cmd/kubeadm/deletetag.sh

diff --git a/.bash_history b/.bash_history
new file mode 100644
index 0000000000..7b43c851e7
--- /dev/null
+++ b/.bash_history
@@ -0,0 +1,6 @@
+cd /go/src/k8s.io/kubernetes/
+ git clone https://github.com/fanux/LVScare /go/src/github.com/fanux/LVScare
+cd /go/src/k8s.io/kubernetes/
+KUBE_GIT_TREE_STATE="clean" KUBE_GIT_VERSION=v1.15.0 KUBE_BUILD_PLATFORMS=linux/amd64 make all WHAT=cmd/kubeadm GOFLAGS=-v
+KUBE_GIT_TREE_STATE="clean" KUBE_GIT_VERSION=v1.15.0 KUBE_BUILD_PLATFORMS=linux/amd64 make all WHAT=cmd/kubeadm GOFLAGS=-v
+exit
diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 0000000000..b7c20a35d8
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,27 @@
+kind: pipeline
+name: default
+workspace:
+    base: /go
+    path: src/k8s.io/kubernetes
+
+steps:
+- name: build
+  image: fanux/kube-build:v1.12.1-2 
+  environment: 
+    GO111MODULE: on
+  commands:
+      - git clone https://github.com/fanux/LVScare /usr/local/go/src/github.com/fanux/LVScare
+      - KUBE_GIT_TREE_STATE="clean" KUBE_GIT_VERSION=v1.15.2 KUBE_BUILD_PLATFORMS=linux/amd64 make all WHAT=cmd/kubeadm GOFLAGS=-v
+      - ls  _output/bin/
+
+- name: publish
+  image: plugins/github-release
+  settings:
+    api_key: 
+        from_secret: git-release-token
+    files: _output/bin/kubeadm
+    title: ${DRONE_TAG}
+    note: Note.md
+    when:
+        event: tag
+
diff --git a/Note.md b/Note.md
new file mode 100644
index 0000000000..5878c67ff7
--- /dev/null
+++ b/Note.md
@@ -0,0 +1,86 @@
+# This is a super kubeadm, support master HA with LVS loadbalance!
+```
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+kubernetesVersion: v1.14.0
+controlPlaneEndpoint: "apiserver.cluster.local:6443" # apiserver DNS name
+apiServer:
+        certSANs:
+        - 127.0.0.1
+        - apiserver.cluster.local
+        - 172.20.241.205
+        - 172.20.241.206
+        - 172.20.241.207
+        - 172.20.241.208
+        - 10.103.97.1          # virturl ip
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: "ipvs"
+ipvs:
+        excludeCIDRs: 
+        - "10.103.97.1/32" # if you don't add this kube-proxy will clean your ipvs rule(kube-proxy still remove it)
+```
+## On master0 10.103.97.100
+```
+echo "10.103.97.100 apiserver.cluster.local" >> /etc/hosts
+kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs  
+mkdir ~/.kube && cp /etc/kubernetes/admin.conf ~/.kube/config
+kubectl apply -f https://docs.projectcalico.org/v3.6/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
+```
+
+## On master1 10.103.97.101
+```
+echo "10.103.97.100 apiserver.cluster.local" >> /etc/hosts
+kubeadm join 10.103.97.100:6443 --token 9vr73a.a8uxyaju799qwdjv \
+    --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 \
+    --experimental-control-plane \
+    --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07 
+
+sed "s/10.103.97.100/10.103.97.101/g" -i /etc/hosts  # if you don't change this, kubelet and kube-proxy will also using master0 apiserver, when master0 down, everything dead!
+```
+
+## On master2 10.103.97.102
+```
+echo "10.103.97.100 apiserver.cluster.local" >> /etc/hosts
+kubeadm join 10.103.97.100:6443 --token 9vr73a.a8uxyaju799qwdjv \
+    --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 \
+    --experimental-control-plane \
+    --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07  
+
+sed "s/10.103.97.100/10.103.97.101/g" -i /etc/hosts
+```
+
+## On your nodes
+Join your nodes with local LVS LB 
+```
+echo "10.103.97.1 apiserver.cluster.local" >> /etc/hosts   # using vip
+kubeadm join 10.103.97.1:6443 --token 9vr73a.a8uxyaju799qwdjv \
+    --master 10.103.97.100:6443 \
+    --master 10.103.97.101:6443 \
+    --master 10.103.97.102:6443 \
+    --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866
+```
+Life is much easier!   
+
+# Architecture
+```
+  +----------+                       +---------------+  virturl server: 127.0.0.1:6443
+  | mater0   |<----------------------| ipvs nodes    |    real servers:
+  +----------+                      |+---------------+            10.103.97.100:6443
+                                    |                             10.103.97.101:6443
+  +----------+                      |                             10.103.97.102:6443
+  | mater1   |<---------------------+
+  +----------+                      |
+                                    |
+  +----------+                      |
+  | mater2   |<---------------------+
+  +----------+
+```
+
+Every node config a ipvs for masters LB.
+
+Then run a lvscare as a staic pod to check realserver is aviliable. `/etc/kubernetes/manifests/sealyun-lvscare.yaml`
+
+# [LVScare](https://github.com/sealyun/LVScare)
+A lvs for kubernetes masters
diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go
index 53f6e2d24e..c92e1e2466 100644
--- a/cmd/kubeadm/app/cmd/join.go
+++ b/cmd/kubeadm/app/cmd/join.go
@@ -41,6 +41,7 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
+	locallb "k8s.io/kubernetes/cmd/kubeadm/app/localLB"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
@@ -158,12 +159,20 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 		Short: "Run this on any machine you wish to join an existing cluster",
 		Long:  joinLongDescription,
 		Run: func(cmd *cobra.Command, args []string) {
-
 			c, err := joinRunner.InitData(args)
 			kubeadmutil.CheckErr(err)
 
 			data := c.(*joinData)
 
+			//sealyun lvscare, only nodes needs this
+			if data.cfg.ControlPlane == nil {
+				fmt.Println("This is not a control plan")
+				if len(locallb.LVScare.Masters) != 0 {
+					locallb.CreateLocalLB(args[0])
+				}
+			} else {
+				fmt.Println("This is a control plan")
+			}
 			err = joinRunner.Run(args)
 			kubeadmutil.CheckErr(err)
 
@@ -182,6 +191,9 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 				joinControPlaneDoneTemp.Execute(data.outputWriter, ctx)
 
 			} else {
+				if len(locallb.LVScare.Masters) != 0 {
+					locallb.LVScareStaticPodToDisk("/etc/kubernetes/manifests")
+				}
 				// otherwise, if the node joined as a worker node;
 				// outputs the join done message and exit
 				fmt.Fprint(data.outputWriter, joinWorkerNodeDoneMsg)
@@ -267,6 +279,16 @@ func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
 		&joinOptions.ignorePreflightErrors, options.IgnorePreflightErrors, joinOptions.ignorePreflightErrors,
 		"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
 	)
+	//sealyun lvscare
+	flagSet.StringSliceVar(
+		&locallb.LVScare.Masters, "master", []string{},
+		"A list of ha masters, --master 192.168.0.2:6443  --master 192.168.0.2:6443  --master 192.168.0.2:6443",
+	)
+	flagSet.StringVar(
+		&locallb.LVScare.Image, "lvscare-image", "fanux/lvscare:latest",
+		"define lvscare image",
+	)
+
 	flagSet.StringVar(
 		&joinOptions.token, options.TokenStr, "",
 		"Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.",
diff --git a/cmd/kubeadm/app/localLB/local_lb.go b/cmd/kubeadm/app/localLB/local_lb.go
new file mode 100644
index 0000000000..f37afa1d58
--- /dev/null
+++ b/cmd/kubeadm/app/localLB/local_lb.go
@@ -0,0 +1,98 @@
+package locallb
+
+import (
+	"fmt"
+
+	"github.com/fanux/LVScare/service"
+	"github.com/fanux/LVScare/utils"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
+)
+
+//LVScare  is
+var LVScare Config
+
+//Config is local lb config
+type Config struct {
+	VirturlServer string // default is 127.0.0.1:6443
+	Masters       []string
+	Image         string   // default is fanux/lvscare:latest
+	Command       []string // [lvscare care --vs 10.103.97.12:6443 --rs 127.0.0.1:8081 --rs 127.0.0.1:8082 --rs 127.0.0.1:8083 --health-path / --health-schem http]
+}
+
+func getSealyunLVScarePod() v1.Pod {
+	v := make(map[string]v1.Volume)
+	t := true
+	pod := staticpod.ComponentPod(v1.Container{
+		Name:            "kube-sealyun-lvscare",
+		Image:           LVScare.Image,
+		ImagePullPolicy: v1.PullIfNotPresent,
+		Command:         LVScare.Command,
+		SecurityContext: &v1.SecurityContext{Privileged: &t},
+	}, v)
+	pod.Spec.HostNetwork = true
+	return pod
+}
+
+//LVScareStaticPodToDisk is
+func LVScareStaticPodToDisk(manifests string) {
+	staticpod.WriteStaticPodToDisk("kube-sealyun-lvscare", manifests, getSealyunLVScarePod())
+}
+
+//InitConfig is
+func InitConfig(vs string) {
+	LVScare.VirturlServer = vs
+	LVScare.Command = []string{
+		"/usr/bin/lvscare",
+		"care",
+		"--vs",
+		LVScare.VirturlServer,
+		"--health-path",
+		"/healthz",
+		"--health-schem",
+		"https",
+	}
+
+	for _, m := range LVScare.Masters {
+		LVScare.Command = append(LVScare.Command, "--rs", m)
+	}
+
+	fmt.Printf("lvscare command is: %s\n", LVScare.Command)
+}
+
+//CreateLVSFirstTime is
+func CreateLVSFirstTime() {
+	vs := LVScare.VirturlServer
+	rs := LVScare.Masters
+
+	lvs, err := service.BuildLvscare(vs, rs)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	//check virturl server
+	service, _ := lvs.GetVirtualServer()
+	if service == nil {
+		lvs.CreateVirtualServer()
+	}
+
+	//check real server
+	//lvs.CheckRealServers("/healthz", "https")
+
+	for _, r := range rs {
+		rip, rport := utils.SplitServer(r)
+		if rip == "" || rport == "" {
+			fmt.Println("real server ip and port is null")
+		}
+		lvs.AddRealServer(rip, rport)
+	}
+
+	fmt.Println("creat ipvs first time", vs, rs)
+}
+
+//CreateLocalLB is
+func CreateLocalLB(vs string) {
+	InitConfig(vs)
+	CreateLVSFirstTime()
+	//LVScareStaticPodToDisk(manifests)
+}
diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
index e1d77e1260..5504086471 100644
--- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
+++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
@@ -54,6 +54,7 @@ const (
 	// RSAPrivateKeyBlockType is a possible value for pem.Block.Type.
 	RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
 	rsaKeySize             = 2048
+	duration365d           = time.Hour * 24 * 365
 )
 
 // NewCertificateAuthority creates new certificate and private key for the certificate authority
@@ -571,7 +572,7 @@ func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certifi
 		IPAddresses:  cfg.AltNames.IPs,
 		SerialNumber: serial,
 		NotBefore:    caCert.NotBefore,
-		NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity).UTC(),
+		NotAfter:     time.Now().Add(duration365d * 99).UTC(),
 		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:  cfg.Usages,
 	}
diff --git a/cmd/kubeadm/deletetag.sh b/cmd/kubeadm/deletetag.sh
new file mode 100644
index 0000000000..d0237f615f
--- /dev/null
+++ b/cmd/kubeadm/deletetag.sh
@@ -0,0 +1,2 @@
+git tag -d $1
+git push origin :refs/tags/$1
diff --git a/cmd/kubeadm/kubeadm.go b/cmd/kubeadm/kubeadm.go
index 1b3d58fa17..759c631a76 100644
--- a/cmd/kubeadm/kubeadm.go
+++ b/cmd/kubeadm/kubeadm.go
@@ -30,5 +30,9 @@ func main() {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)
 	}
+	fmt.Println("\n\n")
+	fmt.Println("kubernetes HA install: https://github.com/fanux/sealos")
+	fmt.Println("www.sealyun.com")
+	fmt.Println("\n\n")
 	os.Exit(0)
 }
diff --git a/go.mod b/go.mod
index 8198da04b0..86c405615f 100644
--- a/go.mod
+++ b/go.mod
@@ -57,6 +57,7 @@ require (
 	github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect
 	github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d
+	github.com/fanux/LVScare/service v1.0.1
 	github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-openapi/loads v0.17.2
diff --git a/staging/src/k8s.io/client-go/util/cert/cert.go b/staging/src/k8s.io/client-go/util/cert/cert.go
index 9fd097af5e..bdb9febe16 100644
--- a/staging/src/k8s.io/client-go/util/cert/cert.go
+++ b/staging/src/k8s.io/client-go/util/cert/cert.go
@@ -63,7 +63,7 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro
 			Organization: cfg.Organization,
 		},
 		NotBefore:             now.UTC(),
-		NotAfter:              now.Add(duration365d * 10).UTC(),
+		NotAfter:              now.Add(duration365d * 99).UTC(),
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 		BasicConstraintsValid: true,
 		IsCA:                  true,
@@ -121,7 +121,7 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
 			CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()),
 		},
 		NotBefore: validFrom,
-		NotAfter:  validFrom.Add(maxAge),
+		NotAfter:  validFrom.Add(maxAge * 99),
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 		BasicConstraintsValid: true,
@@ -149,7 +149,7 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
 			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
 		},
 		NotBefore: validFrom,
-		NotAfter:  validFrom.Add(maxAge),
+		NotAfter:  validFrom.Add(maxAge * 99),
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-- 
2.20.1 (Apple Git-117)