Skip to content

Commit

Permalink
Add support for k8s sec group (#17)
Browse files Browse the repository at this point in the history 
* Add support for k8s sec group

Allow all inbound for ports described in
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#check-required-ports

Change CreateSecurityGroup to accept PortRanges

* Fix sec group releated issues
  • Loading branch information
@outcatcher
outcatcher authored Apr 1, 2020
1 parent 102e3c6 commit 1209824
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 27 deletions.
78 changes: 68 additions & 10 deletions driver/opentelekomcloud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
)

const (
dockerPort = 2376
driverName = "otc"
defaultSecurityGroup = "docker-machine-grp"
defaultAZ = "eu-de-03"
Expand All @@ -49,6 +50,19 @@ const (
defaultAuthURL = "https://iam.eu-de.otc.t-systems.com/v3"
defaultVpcName = "vpc-docker-machine"
defaultSubnetName = "subnet-docker-machine"
k8sGroupName = "sg-k8s"
)

var (
// https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#check-required-ports
k8sPorts = []services.PortRange{
// control-plane node(s)
{From: 6443},
{From: 2379, To: 2380},
{From: 10250, To: 10252},
// worker node(s)
{From: 30000, To: 32767},
}
)

type managedSting struct {
Expand Down Expand Up @@ -87,6 +101,8 @@ type Driver struct {
SecurityGroupIDs []string `json:"-"`
ManagedSecurityGroup string `json:"-"`
ManagedSecurityGroupID string `json:"managed_security_group,omitempty"`
K8sSecurityGroup string `json:"-"`
K8sSecurityGroupID string `json:"k8s_security_group,omitempty"`
FloatingIP managedSting `json:"floating_ip"`
Token string `json:"token,omitempty"`
IPVersion int `json:"-"`
Expand Down Expand Up @@ -128,12 +144,26 @@ func (d *Driver) createSubnet() error {
}
return nil
}
func (d *Driver) createK8sGroup() error {
if d.K8sSecurityGroupID != "" || d.K8sSecurityGroup == "" {
return nil
}
sg, err := d.client.CreateSecurityGroup(d.K8sSecurityGroup, k8sPorts...)
if err != nil {
return err
}
d.K8sSecurityGroupID = sg.ID
return nil
}

func (d *Driver) createSecGroup() error {
func (d *Driver) createDefaultGroup() error {
if d.ManagedSecurityGroupID != "" || d.ManagedSecurityGroup == "" {
return nil
}
sg, err := d.client.CreateSecurityGroup(d.ManagedSecurityGroup, d.SSHPort)
sg, err := d.client.CreateSecurityGroup(d.ManagedSecurityGroup,
services.PortRange{From: d.SSHPort},
services.PortRange{From: dockerPort},
)
if err != nil {
return err
}
Expand Down Expand Up @@ -205,9 +235,13 @@ func (d *Driver) createResources() error {
if err := d.createSubnet(); err != nil {
return err
}
if err := d.createSecGroup(); err != nil {
if err := d.createDefaultGroup(); err != nil {
return err
}
if err := d.createK8sGroup(); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -477,6 +511,10 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag {
Name: "otc-skip-default-sg",
Usage: "Don't create default security group",
},
mcnflag.BoolFlag{
Name: "otc-k8s-group",
Usage: "Create security group with k8s ports allowed",
},
}
}

Expand Down Expand Up @@ -508,7 +546,7 @@ func (d *Driver) GetURL() (string, error) {
if err != nil || ip == "" {
return "", err
}
return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(services.DockerPort))), nil
return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(dockerPort))), nil
}

func (d *Driver) GetState() (state.State, error) {
Expand Down Expand Up @@ -613,31 +651,47 @@ func (d *Driver) deleteVPC() error {
return nil
}

func (d *Driver) deleteSecGroups() error {
if err := d.initCompute(); err != nil {
return err
}
for _, id := range []string{d.ManagedSecurityGroupID, d.K8sSecurityGroupID} {
if id == "" {
continue
}
if err := d.client.DeleteSecurityGroup(id); err != nil {
return err
}
if err := d.client.WaitForGroupDeleted(id); err != nil {
return err
}
}
return nil
}

func (d *Driver) Remove() error {
if err := d.Authenticate(); err != nil {
return err
}
if err := d.deleteInstance(); err != nil {
return err
}
if d.ManagedSecurityGroupID != "" {
if err := d.client.DeleteSecurityGroup(d.ManagedSecurityGroupID); err != nil {
return err
}
}
if d.KeyPairName.DriverManaged {
if err := d.client.DeleteKeyPair(d.KeyPairName.Value); err != nil {
return err
}
}
if d.FloatingIP.DriverManaged {
if d.FloatingIP.DriverManaged && d.FloatingIP.Value != "" {
if err := d.client.DeleteFloatingIP(d.FloatingIP.Value); err != nil {
return err
}
}
if err := d.deleteSubnet(); err != nil {
return err
}
if err := d.deleteSecGroups(); err != nil {
return err
}
if err := d.deleteVPC(); err != nil {
return err
}
Expand Down Expand Up @@ -792,6 +846,10 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
d.ManagedSecurityGroup = defaultSecurityGroup
}

if flags.Bool("otc-k8s-group") {
d.K8sSecurityGroup = k8sGroupName
}

d.SetSwarmConfigFromFlags(flags)
return d.checkConfig()
}
Expand Down
11 changes: 8 additions & 3 deletions driver/opentelekomcloud_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,10 @@ func cleanupResources(driver *Driver) error {
if err != nil {
return err
}
if err := driver.client.DeleteFloatingIP(driver.FloatingIP.Value); err != nil {
log.Error(err)
if driver.FloatingIP.DriverManaged && driver.FloatingIP.Value != "" {
if err := driver.client.DeleteFloatingIP(driver.FloatingIP.Value); err != nil {
log.Error(err)
}
}
if instanceID != "" {
driver.InstanceID = instanceID
Expand Down Expand Up @@ -156,6 +158,9 @@ func cleanupResources(driver *Driver) error {
if driver.ManagedSecurityGroupID != "" {
_ = driver.client.DeleteSecurityGroup(driver.ManagedSecurityGroupID)
}
if driver.K8sSecurityGroupID != "" {
_ = driver.client.DeleteSecurityGroup(driver.K8sSecurityGroupID)
}
vpcID, _ := driver.client.FindVPC(vpcName)
if vpcID == "" {
return nil
Expand All @@ -176,7 +181,7 @@ func TestDriver_CreateWithExistingSecGroups(t *testing.T) {
require.NoError(t, err)
require.NoError(t, preDriver.initCompute())
newSG := services.RandomString(10, "nsg-")
sg, err := preDriver.client.CreateSecurityGroup(newSG, 24)
sg, err := preDriver.client.CreateSecurityGroup(newSG, services.PortRange{From: 24})
assert.NoError(t, err)
defer func() {
assert.NoError(t, preDriver.client.DeleteSecurityGroup(sg.ID))
Expand Down
48 changes: 36 additions & 12 deletions driver/services/compute.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,36 +243,43 @@ func (c *Client) FindImage(imageName string) (string, error) {
const (
cidrAll = "0.0.0.0/0"
tcpProtocol = "TCP"
// port used by docker by default
DockerPort = 2376
)

func (c *Client) addInboundRule(secGroupID string, sshPort int) error {
func (c *Client) addInboundRule(secGroupID string, fromPort int, toPort int) error {

ruleOpts := secgroups.CreateRuleOpts{
ParentGroupID: secGroupID,
FromPort: sshPort,
ToPort: sshPort,
FromPort: fromPort,
ToPort: toPort,
CIDR: cidrAll,
IPProtocol: tcpProtocol,
}
return secgroups.CreateRule(c.ComputeV2, ruleOpts).Err
}

// PortRange is simple sec rule port range container
type PortRange struct {
From int
To int
}

// CreateSecurityGroup creates new sec group and returns group ID
func (c *Client) CreateSecurityGroup(securityGroupName string, sshPort int) (*secgroups.SecurityGroup, error) {
func (c *Client) CreateSecurityGroup(securityGroupName string, ports ...PortRange) (*secgroups.SecurityGroup, error) {
opts := secgroups.CreateOpts{
Name: securityGroupName,
Description: "Docker Machine security group",
Description: "Automatically created by docker-machine for OTC",
}
sg, err := secgroups.Create(c.ComputeV2, opts).Extract()
if err != nil {
return nil, err
}
if err := c.addInboundRule(sg.ID, sshPort); err != nil {
return nil, err
}
if err := c.addInboundRule(sg.ID, DockerPort); err != nil {
return nil, err
for _, port := range ports {
if port.To == 0 {
port.To = port.From
}
if err := c.addInboundRule(sg.ID, port.From, port.To); err != nil {
return nil, err
}
}
return sg, nil
}
Expand Down Expand Up @@ -323,6 +330,23 @@ func (c *Client) DeleteSecurityGroup(securityGroupID string) error {
return secgroups.Delete(c.ComputeV2, securityGroupID).Err
}

// WaitForGroupDeleted polls sec group until it returns 404
func (c *Client) WaitForGroupDeleted(securityGroupID string) error {
err := golangsdk.WaitFor(60, func() (b bool, e error) {
err := secgroups.Get(c.ComputeV2, securityGroupID).Err
if err == nil {
return false, nil
}
switch err.(type) {
case golangsdk.ErrDefault404:
return true, nil
default:
return true, err
}
})
return err
}

const floatingIPPoolID = "admin_external_net"

// CreateFloatingIP creates new floating IP in `admin_external_net` pool
Expand Down
4 changes: 2 additions & 2 deletions driver/services/compute_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestClient_CreateSecurityGroup(t *testing.T) {
cleanupResources(t)

client := computeClient(t)
sg, err := client.CreateSecurityGroup(sgName, 22)
sg, err := client.CreateSecurityGroup(sgName, PortRange{From: 22})
require.NoError(t, err)

sgIDs, err := client.FindSecurityGroups([]string{sgName})
Expand Down Expand Up @@ -178,7 +178,7 @@ func TestClient_CreateInstance(t *testing.T) {
require.NoError(t, err)
defer func() { _ = client.DeleteFloatingIP(ip) }()

sg, err := client.CreateSecurityGroup(sgName, 22)
sg, err := client.CreateSecurityGroup(sgName, PortRange{From: 22})
require.NoError(t, err)
defer func() { _ = client.DeleteSecurityGroup(sg.ID) }()

Expand Down

0 comments on commit 1209824

Please sign in to comment.