diff --git a/compare-slugs-tool/.gitignore b/compare-slugs-tool/.gitignore new file mode 100644 index 0000000..51e7f08 --- /dev/null +++ b/compare-slugs-tool/.gitignore @@ -0,0 +1,4 @@ +fixtures/ +data_in_da_and_slug_mapping_not_in_sfdc.csv +data_not_in_slug_mapping_and_sfdc.csv +.env \ No newline at end of file diff --git a/compare-slugs-tool/README.md b/compare-slugs-tool/README.md new file mode 100644 index 0000000..8af1b58 --- /dev/null +++ b/compare-slugs-tool/README.md @@ -0,0 +1,27 @@ +# Compare Slugs Tool +Purpose: To get a list of the Insights project slugs that have ES indices but those that don't exist in SFDC. + + - Get all the unique enriched ES indices. Except --for-merge and any index that is an alias + - Get their project slugs from the index name + - Check slug_mapping if the da_name exists in the table + - List down the slugs for which slug_mapping entry does not exist. + +## How to run the tool + - clone v1 repo from + - copy the fixtures files of the v1 repo from to `./compare-slugs-tool/fixtures` + - run `$ go run main.go` + - two files will be generated: + * 1.`data_in_da_and_slug_mapping_not_in_sfdc.csv` will contain project data for projects in insights and in the slug mapping table but are NOT in SFDC. + * 2.`data_not_in_slug_mapping_and_sfdc.csv` will contain project data for projects in insights fixtures but NOT in the slug mapping table and SFDC. + + - import the csv files into an Excel sheet for better readability + +## Required Environment Variables + +`ES_URL`= https://username:password@fqdn:port + +`TOKEN` = auth0-token (Bearer + token) + +`PROJECTS_SERVICE_BASE_URL` = https://api-gw.platform.linuxfoundation.org/project-service/v1/ + +`SH_DB` = username:password@tcp(host:port)/database_name?charset=utf8 \ No newline at end of file diff --git a/compare-slugs-tool/main.go b/compare-slugs-tool/main.go new file mode 100644 index 0000000..22a9ede --- /dev/null +++ b/compare-slugs-tool/main.go @@ -0,0 +1,364 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + http2 "github.com/LF-Engineering/dev-analytics-libraries/http" + _ "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + "gopkg.in/yaml.v2" +) + +func main() { + fileList := make([]string, 0) + err := filepath.Walk(".", func(path string, f os.FileInfo, err error) error { + if ok := strings.HasSuffix(path, ".yaml"); ok { + fileList = append(fileList, path) + } + return err + }) + + if err != nil { + log.Fatal(err) + } + + esIndices := make(map[string]map[string]map[string]interface{}) + if len(os.Getenv("ES_URL")) == 0 { + log.Fatal("ES_URL environment variable not set") + } + requestURL := fmt.Sprintf("%s%s", os.Getenv("ES_URL"), "/_aliases?pretty=true") + resp, err := http.Get(requestURL) + if err != nil { + log.Fatal(err) + } + + defer func() { + if err := resp.Body.Close(); err != nil { + log.Fatal(err.Error()) + } + }() + + err = json.NewDecoder(resp.Body).Decode(&esIndices) + if err != nil { + log.Fatal(err) + } + fmt.Println("len(esIndices)", len(esIndices)) + + finalObject := make(map[string]map[string]bool) + + projects, err := getProjectsFromProjectsService() + if err != nil { + log.Fatal(err) + } + + sfProjects := make(map[string]project) + for _, proj := range projects { + sfProjects[proj.Slug] = proj + } + + for k, aliases := range esIndices { + if strings.HasSuffix(strings.TrimSpace(k), "github-pull_request") { + continue + } + + if strings.HasSuffix(strings.TrimSpace(k), "github-repository") { + continue + } + + if strings.HasSuffix(strings.TrimSpace(k), "-raw") { + continue + } + + aliases := aliases["aliases"] + if aliases != nil { + for alias := range aliases { + finalObject[alias] = make(map[string]bool, 0) + } + } + + finalObject[k] = make(map[string]bool, 0) + } + + config := RDSConfig{ + URL: os.Getenv("SH_DB"), + } + + dbConn, err := ConnectDatabase(&config) + if err != nil { + log.Fatal(err) + } + dbService := New(dbConn) + + mappings := make(map[string]SlugMapping) + slugMapping, err := dbService.GetSlugMapping() + if err != nil { + log.Fatal(err) + } + + for _, mapping := range slugMapping { + mappings[mapping.DAName] = mapping + } + + finalOutputInInsightsAndSlugMappingNotInSfdc := make([]OutputStruct, 0) + finalOutputNotInSlugMappingAndSfdc := make([]OutputStruct, 0) + var DASlugs []string + for _, file := range fileList { + yamlFile, err := ioutil.ReadFile(file) + if err != nil { + log.Printf("yamlFile.Get err #%v ", err) + } + + var c Fixture + err = yaml.Unmarshal(yamlFile, &c) + if err != nil { + log.Println(file) + log.Fatalf("Unmarshal: %v", err) + } + DASlugs = append(DASlugs, c.Native.Slug) + + for _, datasource := range c.DataSources { + projSlugFormatted := strings.ReplaceAll(c.Native.Slug, "/", "-") + datasourceSlugFormatted := strings.ReplaceAll(datasource.Slug, "/", "-") + index := fmt.Sprintf("sds-%s-%s", projSlugFormatted, datasourceSlugFormatted) + + if _, ok := finalObject[index]; !ok { + if strings.HasSuffix(strings.TrimSpace(index), "github-pull_request") { + continue + } + + if strings.HasSuffix(strings.TrimSpace(index), "github-repository") { + continue + } + + if strings.HasSuffix(strings.TrimSpace(index), "-raw") { + continue + } + // log indices we don't have in elastic search + fmt.Println("index not in es: ", index) + continue + } + + if _, ok := mappings[c.Native.Slug]; ok { + if _, exists := sfProjects[mappings[c.Native.Slug].SFName]; !exists { + finalOutputInInsightsAndSlugMappingNotInSfdc = append(finalOutputInInsightsAndSlugMappingNotInSfdc, OutputStruct{ + IndexName: index, + InsightsSlug: mappings[c.Native.Slug].DAName, + Disabled: mappings[c.Native.Slug].IsDisabled, + }) + } + } else { + finalOutputNotInSlugMappingAndSfdc = append(finalOutputNotInSlugMappingAndSfdc, OutputStruct{ + IndexName: index, + InsightsSlug: c.Native.Slug, + Disabled: c.Disabled, + }) + } + } + + } + + csvFile, err := os.Create("./data_in_da_and_slug_mapping_not_in_sfdc.csv") + if err != nil { + log.Fatal(err) + } + defer csvFile.Close() + + writer := csv.NewWriter(csvFile) + + for _, line := range finalOutputInInsightsAndSlugMappingNotInSfdc { + var row []string + row = append(row, line.IndexName) + row = append(row, line.InsightsSlug) + row = append(row, strconv.FormatBool(line.Disabled)) + if err := writer.Write(row); err != nil { + log.Fatal(err) + } + } + writer.Flush() + + csvFile, err = os.Create("./data_not_in_slug_mapping_and_sfdc.csv") + if err != nil { + log.Fatal(err) + } + defer csvFile.Close() + + writer = csv.NewWriter(csvFile) + + for _, line := range finalOutputNotInSlugMappingAndSfdc { + var row []string + row = append(row, line.IndexName) + row = append(row, line.InsightsSlug) + row = append(row, strconv.FormatBool(line.Disabled)) + if err := writer.Write(row); err != nil { + log.Fatal(err) + } + } + writer.Flush() +} + +// Fixture struct +type Fixture struct { + Native struct { + Slug string `yaml:"slug"` + } `yaml:"native"` + DataSources []struct { + Slug string `yaml:"slug,omitempty"` + Projects []struct { + Name string `yaml:"name"` + Endpoints []struct { + Name string `yaml:"name"` + } `yaml:"endpoints"` + Flags interface{} `yaml:"flags"` + } `yaml:"projects"` + //Name []string `yaml:"name"` + } `yaml:"data_sources"` + Disabled bool `yaml:"disabled"` +} + +// RDSConfig struct +type RDSConfig struct { + URL string +} + +// ConnectDatabase initializes database connection +func ConnectDatabase(config *RDSConfig) (*sqlx.DB, error) { + var dbConnection *sqlx.DB + + dbConnection, err := sqlx.Connect("mysql", config.URL) + if err != nil { + return nil, err + } + + dbConnection.SetMaxOpenConns(1) // The default is 0 (unlimited) + dbConnection.SetMaxIdleConns(0) // defaultMaxIdleConnections = 2, (0 is unlimited) + dbConnection.SetConnMaxLifetime(0) // 0, connections are reused forever. + + fmt.Println("DB Service Initialized!") + return dbConnection, nil +} + +// service ... +type service struct { + db *sqlx.DB +} + +// SlugMapping struct +type SlugMapping struct { + DAName string `db:"da_name" json:"da_name"` + SfID string `db:"sf_id" json:"sf_id"` + SFName string `db:"sf_name" json:"sf_name"` + IsDisabled bool `db:"is_disabled" json:"is_disabled"` + CreatedAt interface{} `db:"created_at" json:"created_at"` +} + +// New creates new db service instance with given db +func New(db *sqlx.DB) Service { + return &service{ + db: db, + } +} + +// Service ... +type Service interface { + GetSlugMapping() ([]SlugMapping, error) +} + +// GetSlugMapping returns a list of data from the slug_mapping table +func (s *service) GetSlugMapping() (mapping []SlugMapping, err error) { + query := ` + SELECT * + FROM + slug_mapping;` + err = s.db.Select(&mapping, query) + return +} + +// getProjectsFromProjectsService returns a list of all projects from the projects service +func getProjectsFromProjectsService() ([]project, error) { + httpClient := http2.NewClientProvider(60 * time.Second) + if len(os.Getenv("TOKEN")) == 0 { + return nil, errors.New("TOKEN environment variable not set") + } + if len(os.Getenv("PROJECTS_SERVICE_BASE_URL")) == 0 { + return nil, errors.New("PROJECTS_SERVICE_BASE_URL environment variable not set") + } + headers := make(map[string]string) + headers["Accept"] = "application/json" + headers["Authorization"] = os.Getenv("TOKEN") + + url := fmt.Sprintf("%s%s", os.Getenv("PROJECTS_SERVICE_BASE_URL"), "projects?pageSize=100000") + _, resp, err := httpClient.Request(url, http.MethodGet, headers, nil, nil) + if err != nil { + return nil, err + } + + var projects Data + err = json.Unmarshal(resp, &projects) + if err != nil { + return nil, err + } + return projects.Data, nil +} + +// project struct +type project struct { + AutoJoinEnabled bool `json:"AutoJoinEnabled" yaml:"AutoJoinEnabled"` + Description string `json:"Description" yaml:"Description"` + Parent string `json:"Parent" yaml:"Parent"` + Name string `json:"Name" yaml:"Name"` + ProjectLogo string `json:"ProjectLogo" yaml:"ProjectLogo"` + RepositoryURL string `json:"RepositoryURL" yaml:"RepositoryURL"` + Slug string `json:"Slug" yaml:"Slug"` + StartDate string `json:"StartDate" yaml:"StartDate"` + Status string `json:"Status" yaml:"Status"` + Website string `json:"Website" yaml:"Website"` + Category string `json:"Category" yaml:"Category"` + CreatedDate string `json:"CreatedDate" yaml:"CreatedDate"` + EndDate string `json:"EndDate" yaml:"EndDate"` + Foundation struct { + ID string `json:"ID" yaml:"ID"` + LogoURL string `json:"LogoURL" yaml:"LogoURL"` + Name string `json:"Name" yaml:"Name"` + } `json:"Foundation" yaml:"Foundation"` + ID string `json:"ID" yaml:"ID"` + ModifiedDate string `json:"ModifiedDate" yaml:"ModifiedDate"` + OpportunityOwner struct { + Email string `json:"Email" yaml:"Email"` + FirstName string `json:"FirstName" yaml:"FirstName"` + ID string `json:"ID" yaml:"ID"` + LastName string `json:"LastName" yaml:"LastName"` + } `json:"OpportunityOwner" yaml:"OpportunityOwner"` + Owner struct { + Email string `json:"Email" yaml:"Email"` + FirstName string `json:"FirstName" yaml:"FirstName"` + ID string `json:"ID" yaml:"ID"` + LastName string `json:"LastName" yaml:"LastName"` + } `json:"Owner" yaml:"Owner"` + ProjectType string `json:"ProjectType" yaml:"ProjectType"` + SystemModStamp string `json:"SystemModStamp" yaml:"SystemModStamp"` + Type string `json:"Type" yaml:"Type"` + Projects []project `json:"Projects" yaml:"Projects"` +} + +// Data struct +type Data struct { + Data []project `json:"Data"` +} + +// OutputStruct struct +type OutputStruct struct { + IndexName string + InsightsSlug string + Disabled bool +} diff --git a/go.mod b/go.mod index a60cf88..6a4603d 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,10 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/elastic/go-elasticsearch/v8 v8.0.0-20201229214741-2366c2514674 github.com/go-git/go-git/v5 v5.2.0 + github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/google/go-github/v33 v33.0.0 github.com/google/uuid v1.1.4 + github.com/jmoiron/sqlx v1.3.4 // indirect github.com/json-iterator/go v1.1.10 github.com/labstack/gommon v0.3.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 76fcd46..4be7eb4 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,9 @@ github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homk github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -163,6 +166,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -177,11 +182,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=