Merge pull request #1 from thefeli73/basic-functionality

AWS Indexing
This commit is contained in:
Felix Schulze 2023-03-22 14:29:05 +01:00 committed by GitHub
commit bd976c1791
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 316 additions and 47 deletions

7
.gitignore vendored
View File

@ -13,3 +13,10 @@
# Dependency directories (remove the comment below to include it)
# vendor/
#protect secrets
*.env
*.credentials
config.yaml

5
config.default.yaml Normal file
View File

@ -0,0 +1,5 @@
mtd:
services: []
aws:
regions: []
credentials_path: ./mtdaws/.credentials

22
go.mod
View File

@ -2,6 +2,24 @@ module github.com/thefeli73/polemos
go 1.20
require github.com/aws/aws-sdk-go v1.44.220
require (
github.com/aws/aws-sdk-go-v2 v1.17.6
github.com/aws/aws-sdk-go-v2/config v1.18.17
github.com/aws/aws-sdk-go-v2/service/ec2 v1.90.0
github.com/google/uuid v1.3.0
gopkg.in/yaml.v3 v3.0.1
)
require github.com/jmespath/go-jmespath v0.4.0 // indirect
require (
github.com/aws/aws-sdk-go-v2/credentials v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
)

63
go.sum
View File

@ -1,42 +1,45 @@
github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo=
github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.17.6 h1:Y773UK7OBqhzi5VDXMi1zVGsoj+CVHs2eaC2bDsLwi0=
github.com/aws/aws-sdk-go-v2 v1.17.6/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/config v1.18.17 h1:jwTkhULSrbr/SQA8tfdYqZxpG8YsRycmIXxJcbrqY5E=
github.com/aws/aws-sdk-go-v2/config v1.18.17/go.mod h1:Lj3E7XcxJnxMa+AYo89YiL68s1cFJRGduChynYU67VA=
github.com/aws/aws-sdk-go-v2/credentials v1.13.17 h1:IubQO/RNeIVKF5Jy77w/LfUvmmCxTnk2TP1UZZIMiF4=
github.com/aws/aws-sdk-go-v2/credentials v1.13.17/go.mod h1:K9xeFo1g/YPMguMUD69YpwB4Nyi6W/5wn706xIInJFg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 h1:/2Cb3SK3xVOQA7Xfr5nCWCo5H3UiNINtsVvVdk8sQqA=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0/go.mod h1:neYVaeKr5eT7BzwULuG2YbLhzWZ22lpjKdCybR7AXrQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 h1:y+8n9AGDjikyXoMBTRaHHHSaFEB8267ykmvyPodJfys=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 h1:r+Kv+SEJquhAZXaJ7G4u44cIwXV3f8K+N482NNAzJZA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 h1:hf+Vhp5WtTdcSdE+yEcUz8L73sAzN0R+0jQv+Z51/mI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31/go.mod h1:5zUjguZfG5qjhG9/wqmuyHRyUftl2B5Cp6NNxNC6kRA=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.90.0 h1:oRl2nzkuU/qMPvudU3qQ+GUAMV5POP3V/aJTJ7Q0lT0=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.90.0/go.mod h1:zDr1uSSLVYc6KqXvrmqYkeqnfbmOOrbVloz4Eqsc83k=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 h1:c5qGfdbCHav6viBwiyDns3OXqhqAbGjfIB4uVu2ayhk=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24/go.mod h1:HMA4FZG6fyib+NDo5bpIxX1EhYjrAOveZJY2YR0xrNE=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 h1:bdKIX6SVF3nc3xJFw6Nf0igzS6Ff/louGq8Z6VP/3Hs=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.5/go.mod h1:vuWiaDB30M/QTC+lI3Wj6S/zb7tpUK2MSYgy3Guh2L0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 h1:xLPZMyuZ4GuqRCIec/zWuIhRFPXh2UOJdLXBSi64ZWQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5/go.mod h1:QjxpHmCwAg0ESGtPQnLIVp7SedTOBMYy+Slr3IfMKeI=
github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 h1:rIFn5J3yDoeuKCE9sESXqM5POTAhOP1du3bv/qTL+tE=
github.com/aws/aws-sdk-go-v2/service/sts v1.18.6/go.mod h1:48WJ9l3dwP0GSHWGc5sFGGlCkuA82Mc2xnw+T6Q8aDw=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

57
main.go
View File

@ -1,8 +1,61 @@
package main
import (
"github.com/thefeli73/polemos/mtd_aws"
"fmt"
"net/netip"
"github.com/google/uuid"
"github.com/thefeli73/polemos/mtdaws"
"github.com/thefeli73/polemos/state"
)
// ConfigPath is a string of the location for the configfile
var ConfigPath string
func main() {
var _ = mtd_aws.Session()
fmt.Println("Starting Polemos")
ConfigPath = "config.yaml"
config := state.LoadConf(ConfigPath)
state.SaveConf(ConfigPath, config)
config = indexInstances(config)
}
func indexInstances(config state.Config) state.Config {
fmt.Println("Indexing instances")
//index AWS instances
awsInstances := mtdaws.GetInstances(config)
for _, instance := range awsInstances {
cloudID := mtdaws.GetCloudID(instance)
ip, err := netip.ParseAddr(instance.PublicIP)
if err != nil {
fmt.Println("Error converting ip:", err)
continue
}
newService, found := indexInstance(config, cloudID, ip)
if !found {
config.MTD.Services = append(config.MTD.Services, newService)
state.SaveConf(ConfigPath, config)
}
}
return config
}
func indexInstance(config state.Config, cloudID string, serviceIP netip.Addr) (state.Service, bool) {
found := false
for _, service := range config.MTD.Services {
if service.CloudID == cloudID {
found = true
}
}
u := uuid.New()
newService := state.Service{
ID: state.CustomUUID(u),
CloudID: cloudID,
ServiceIP: serviceIP}
return newService, found
}

View File

@ -1,12 +0,0 @@
package mtd_aws
import (
//"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
)
func Session() *session.Session {
session := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
return session
}

102
mtdaws/utils.go Normal file
View File

@ -0,0 +1,102 @@
package mtdaws
import (
"context"
"fmt"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/thefeli73/polemos/state"
)
// AwsInstance is basic info about a single aws instance (instance id, redion, pubIP and privIP)
type AwsInstance struct {
InstanceID string
Region string
PublicIP string
PrivateIP string
}
// NewConfig creates a AWS config for a specific region
func NewConfig(region string, credentials string) aws.Config {
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithSharedConfigFiles([]string{credentials}), config.WithRegion(region))
if err != nil {
fmt.Println("Error creating config:", err)
fmt.Println("Configure Credentials in line with the documentation found here: https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials")
os.Exit(1)
}
return cfg
}
// GetCloudID returns a string to find the instance based on information from aws
func GetCloudID(instance AwsInstance) string {
return "aws_" + instance.Region + "_" + instance.InstanceID
}
// GetInstances scans all configured regions for instances and add them to services
func GetInstances(config state.Config) []AwsInstance {
awsInstances := []AwsInstance{}
for _, region := range config.AWS.Regions {
awsConfig := NewConfig(region, config.AWS.CredentialsPath)
instances, err := Instances(awsConfig)
if err != nil {
fmt.Println("Error listing instances:", err)
continue
}
//fmt.Println("Listing instances in region:", region)
for _, instance := range instances {
var publicAddr string
if instance.PublicIpAddress != nil {
publicAddr = aws.ToString(instance.PublicIpAddress)
}
awsInstances = append(awsInstances, AwsInstance{
InstanceID: aws.ToString(instance.InstanceId),
Region: region,
PublicIP: publicAddr,
PrivateIP: aws.ToString(instance.PrivateIpAddress)})
}
}
return awsInstances
}
// PrintInstanceInfo prints info about a specific instance in a region
func PrintInstanceInfo(instance *types.Instance) {
fmt.Println("\tInstance ID:", aws.ToString(instance.InstanceId))
fmt.Println("\t\tInstance Type:", string(instance.InstanceType))
fmt.Println("\t\tAMI ID:", aws.ToString(instance.ImageId))
fmt.Println("\t\tState:", string(instance.State.Name))
fmt.Println("\t\tAvailability Zone:", aws.ToString(instance.Placement.AvailabilityZone))
if instance.PublicIpAddress != nil {
fmt.Println("\t\tPublic IP Address:", aws.ToString(instance.PublicIpAddress))
}
fmt.Println("\t\tPrivate IP Address:", aws.ToString(instance.PrivateIpAddress))
}
// Instances returns all instances for a config i.e. a region
func Instances(config aws.Config) ([]*types.Instance, error) {
svc := ec2.NewFromConfig(config)
input := &ec2.DescribeInstancesInput{}
var instances []*types.Instance
paginator := ec2.NewDescribeInstancesPaginator(svc, input)
for paginator.HasMorePages() {
page, err := paginator.NextPage(context.TODO())
if err != nil {
return nil, err
}
for _, reservation := range page.Reservations {
for _, instance := range reservation.Instances {
instances = append(instances, &instance)
}
}
}
return instances, nil
}

93
state/config.go Normal file
View File

@ -0,0 +1,93 @@
package state
import (
"fmt"
"io/ioutil"
"net/netip"
"os"
"github.com/google/uuid"
"gopkg.in/yaml.v3"
)
// Config contains all MTD services and cloud provider configs
type Config struct {
MTD mtdconf `yaml:"mtd"`
AWS aws `yaml:"aws"`
}
type mtdconf struct {
Services []Service `yaml:"services"`
}
// Service contains all necessary information about a service to identify it in the cloud as well as configuring a proxy for it
type Service struct {
ID CustomUUID `yaml:"id"`
CloudID string `yaml:"cloud_id"`
EntryIP netip.Addr `yaml:"entry_ip"`
EntryPort uint16 `yaml:"entry_port"`
ServiceIP netip.Addr `yaml:"service_ip"`
ServicePort uint16 `yaml:"service_port"`
}
// CustomUUID is an alias for uuid.UUID to enable custom unmarshal function
type CustomUUID uuid.UUID
type aws struct {
Regions []string `yaml:"regions"`
CredentialsPath string `yaml:"credentials_path"`
}
// UnmarshalYAML parses uuid in yaml to CustomUUID type
func (u *CustomUUID) UnmarshalYAML(value *yaml.Node) error {
id, err := uuid.Parse(value.Value)
if err != nil {
return err
}
*u = CustomUUID(id)
return nil
}
// MarshalYAML parses CustomUUID type to uuid string for yaml
func (u CustomUUID) MarshalYAML() (interface{}, error) {
return uuid.UUID(u).String(), nil
}
// LoadConf loads config from a yaml file
func LoadConf(filename string) (Config) {
var config Config
data, err := ioutil.ReadFile(filename)
if err != nil {
fmt.Println("Error reading file:", err)
fmt.Println("Attempting to load default config")
data, err = ioutil.ReadFile("config.default.yaml")
if err != nil {
fmt.Println("Error reading file:", err)
os.Exit(1)
}
}
err = yaml.Unmarshal([]byte(data), &config)
if err != nil {
fmt.Println("Error importing config:", err)
os.Exit(1)
}
fmt.Println("Imported config succesfully!")
return config
}
// SaveConf saves config to yaml file
func SaveConf(filename string, config Config) (error) {
yamlBytes, err := yaml.Marshal(&config)
if err != nil {
return err
}
err = ioutil.WriteFile(filename, yamlBytes, 0644)
if err != nil {
return err
}
return nil
}