From 6d4fef4b9a27e8a00de598312eb3da79742c035b Mon Sep 17 00:00:00 2001 From: Diego Pontoriero Date: Mon, 18 Dec 2017 23:00:14 -0800 Subject: [PATCH] license: add bill of materials. --- Makefile | 6 +- bill-of-materials.json | 276 +++++++ glide.lock | 6 +- glide.yaml | 4 + .../coreos/license-bill-of-materials/LICENSE | 19 + .../license-bill-of-materials.go | 733 ++++++++++++++++++ 6 files changed, 1041 insertions(+), 3 deletions(-) create mode 100644 bill-of-materials.json create mode 100644 vendor/github.com/coreos/license-bill-of-materials/LICENSE create mode 100644 vendor/github.com/coreos/license-bill-of-materials/license-bill-of-materials.go diff --git a/Makefile b/Makefile index 3bab85d2..f2bd42a7 100644 --- a/Makefile +++ b/Makefile @@ -33,9 +33,10 @@ release-binary: @go build -o /go/bin/dex -v -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/dex .PHONY: revendor -revendor: +revendor: bin/license-bill-of-materials @glide up -v @glide-vc --use-lock-file --no-tests --only-code + @./bin/license-bill-of-materials ./cmd/dex ./cmd/example-app > bill-of-materials.json test: @go test -v -i $(shell go list ./... | grep -v '/vendor/') @@ -75,6 +76,9 @@ bin/protoc: scripts/get-protoc bin/protoc-gen-go: @go install -v $(REPO_PATH)/vendor/github.com/golang/protobuf/protoc-gen-go +bin/license-bill-of-materials: + @CGO_ENABLED=1 go install -v $(REPO_PATH)/vendor/github.com/coreos/license-bill-of-materials + .PHONY: check-go-version check-go-version: @./scripts/check-go-version diff --git a/bill-of-materials.json b/bill-of-materials.json new file mode 100644 index 00000000..13669b8e --- /dev/null +++ b/bill-of-materials.json @@ -0,0 +1,276 @@ +[ + { + "project": "github.com/beevik/etree", + "licenses": [ + { + "type": "BSD 2-clause \"Simplified\" License", + "confidence": 0.9658536585365853 + } + ] + }, + { + "project": "github.com/cockroachdb/cockroach-go/crdb", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9988925802879292 + } + ] + }, + { + "project": "github.com/coreos/dex", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/coreos/etcd", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/coreos/go-oidc", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/ghodss/yaml", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.8357142857142857 + } + ] + }, + { + "project": "github.com/go-sql-driver/mysql", + "licenses": [ + { + "type": "Mozilla Public License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/golang/protobuf", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.92 + } + ] + }, + { + "project": "github.com/gorilla/handlers", + "licenses": [ + { + "type": "BSD 2-clause \"Simplified\" License", + "confidence": 0.9852216748768473 + } + ] + }, + { + "project": "github.com/gorilla/mux", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "github.com/gtank/cryptopasta", + "licenses": [ + { + "type": "Creative Commons Zero v1.0 Universal", + "confidence": 0.9642857142857143 + } + ] + }, + { + "project": "github.com/jonboulle/clockwork", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/lib/pq", + "licenses": [ + { + "type": "MIT License", + "confidence": 0.9891304347826086 + } + ] + }, + { + "project": "github.com/mattn/go-sqlite3", + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] + }, + { + "project": "github.com/pquerna/cachecontrol", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/russellhaering/goxmldsig", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9573241061130334 + } + ] + }, + { + "project": "github.com/sirupsen/logrus", + "licenses": [ + { + "type": "MIT License", + "confidence": 1 + } + ] + }, + { + "project": "github.com/spf13/cobra", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 0.9573241061130334 + } + ] + }, + { + "project": "github.com/spf13/pflag", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "golang.org/x/crypto", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "golang.org/x/net", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "golang.org/x/oauth2", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "golang.org/x/text", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "google.golang.org/genproto/googleapis/rpc/status", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "google.golang.org/grpc", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.979253112033195 + } + ] + }, + { + "project": "gopkg.in/asn1-ber.v1", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "gopkg.in/ldap.v2", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "gopkg.in/square/go-jose.v2", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "gopkg.in/square/go-jose.v2/json", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "gopkg.in/yaml.v2", + "licenses": [ + { + "type": "GNU Lesser General Public License v3.0", + "confidence": 0.9528301886792453 + }, + { + "type": "MIT License", + "confidence": 0.8975609756097561 + } + ] + } +] diff --git a/glide.lock b/glide.lock index c0cb44cc..5ec5d4ed 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: aa8fb290b0e14af4bf4d4cae250ce66561c819a9cdd6447a0432de26f9216318 -updated: 2017-11-30T16:40:26.928155912-08:00 +hash: dda54feb39d5947ad20e9d583ccad9be02343dac0f52752944b7cd39668af7a4 +updated: 2017-12-19T10:23:03.503553615-08:00 imports: - name: github.com/beevik/etree version: 4cd0dd976db869f817248477718071a28e978df0 @@ -20,6 +20,8 @@ imports: - pkg/transport - name: github.com/coreos/go-oidc version: be73733bb8cc830d0205609b95d125215f8e9c70 +- name: github.com/coreos/license-bill-of-materials + version: d70207c33a3c79a1c0479b208f8b7ab6215144c7 - name: github.com/ghodss/yaml version: bea76d6a4713e18b7f5321a2b020738552def3ea - name: github.com/go-sql-driver/mysql diff --git a/glide.yaml b/glide.yaml index eece3c63..51d05c18 100644 --- a/glide.yaml +++ b/glide.yaml @@ -154,3 +154,7 @@ import: version: 4cd0dd976db869f817248477718071a28e978df0 - package: github.com/jonboulle/clockwork version: bcac9884e7502bb2b474c0339d889cb981a2f27f + +# License bill of materials generator. +- package: github.com/coreos/license-bill-of-materials + version: d70207c33a3c79a1c0479b208f8b7ab6215144c7 diff --git a/vendor/github.com/coreos/license-bill-of-materials/LICENSE b/vendor/github.com/coreos/license-bill-of-materials/LICENSE new file mode 100644 index 00000000..2c68650f --- /dev/null +++ b/vendor/github.com/coreos/license-bill-of-materials/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Patrick Mézard + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/coreos/license-bill-of-materials/license-bill-of-materials.go b/vendor/github.com/coreos/license-bill-of-materials/license-bill-of-materials.go new file mode 100644 index 00000000..7ee70685 --- /dev/null +++ b/vendor/github.com/coreos/license-bill-of-materials/license-bill-of-materials.go @@ -0,0 +1,733 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/coreos/license-bill-of-materials/assets" +) + +// Template holds pre-constructed license template info +type Template struct { + Title string + Nickname string + Words map[string]int +} + +func parseTemplate(content string) (*Template, error) { + t := Template{} + text := []byte{} + state := 0 + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if state == 0 { + if line == "---" { + state = 1 + } + } else if state == 1 { + if line == "---" { + state = 2 + } else { + if strings.HasPrefix(line, "title:") { + t.Title = strings.TrimSpace(line[len("title:"):]) + } else if strings.HasPrefix(line, "nickname:") { + t.Nickname = strings.TrimSpace(line[len("nickname:"):]) + } + } + } else if state == 2 { + text = append(text, scanner.Bytes()...) + text = append(text, []byte("\n")...) + } + } + t.Words = makeWordSet(text) + return &t, scanner.Err() +} + +func loadTemplates() ([]*Template, error) { + templates := []*Template{} + for _, a := range assets.Assets { + templ, err := parseTemplate(a.Content) + if err != nil { + return nil, err + } + templates = append(templates, templ) + } + return templates, nil +} + +var ( + reWords = regexp.MustCompile(`[\w']+`) + reCopyright = regexp.MustCompile( + `(?i)\s*Copyright (?:©|\(c\)|\xC2\xA9)?\s*(?:\d{4}|\[year\]).*`) +) + +func cleanLicenseData(data []byte) []byte { + data = bytes.ToLower(data) + data = reCopyright.ReplaceAll(data, nil) + return data +} + +func makeWordSet(data []byte) map[string]int { + words := map[string]int{} + data = cleanLicenseData(data) + matches := reWords.FindAll(data, -1) + for i, m := range matches { + s := string(m) + if _, ok := words[s]; !ok { + // Non-matching words are likely in the license header, to mention + // copyrights and authors. Try to preserve the initial sequences, + // to display them later. + words[s] = i + } + } + return words +} + +// Word holds word and word position in a license +type Word struct { + Text string + Pos int +} + +type sortedWords []Word + +func (s sortedWords) Len() int { + return len(s) +} + +func (s sortedWords) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedWords) Less(i, j int) bool { + return s[i].Pos < s[j].Pos +} + +// MatchResult represents a matched template and matching metrics +type MatchResult struct { + Template *Template + Score float64 + ExtraWords []string + MissingWords []string +} + +func sortAndReturnWords(words []Word) []string { + sort.Sort(sortedWords(words)) + tokens := []string{} + for _, w := range words { + tokens = append(tokens, w.Text) + } + return tokens +} + +// matchTemplates returns the best license template matching supplied data, +// its score between 0 and 1 and the list of words appearing in license but not +// in the matched template. +func matchTemplates(license []byte, templates []*Template) MatchResult { + bestScore := float64(-1) + var bestTemplate *Template + bestExtra := []Word{} + bestMissing := []Word{} + words := makeWordSet(license) + for _, t := range templates { + extra := []Word{} + missing := []Word{} + common := 0 + for w, pos := range words { + _, ok := t.Words[w] + if ok { + common++ + } else { + extra = append(extra, Word{ + Text: w, + Pos: pos, + }) + } + } + for w, pos := range t.Words { + if _, ok := words[w]; !ok { + missing = append(missing, Word{ + Text: w, + Pos: pos, + }) + } + } + score := 2 * float64(common) / (float64(len(words)) + float64(len(t.Words))) + if score > bestScore { + bestScore = score + bestTemplate = t + bestMissing = missing + bestExtra = extra + } + } + return MatchResult{ + Template: bestTemplate, + Score: bestScore, + ExtraWords: sortAndReturnWords(bestExtra), + MissingWords: sortAndReturnWords(bestMissing), + } +} + +// fixEnv returns a copy of the process environment where GOPATH is adjusted to +// supplied value. It returns nil if gopath is empty. +func fixEnv(gopath string) []string { + if gopath == "" { + return nil + } + kept := []string{ + "GOPATH=" + gopath, + } + for _, env := range os.Environ() { + if !strings.HasPrefix(env, "GOPATH=") { + kept = append(kept, env) + } + } + return kept +} + +// MissingError reports on missing licenses +type MissingError struct { + Err string +} + +func (err *MissingError) Error() string { + return err.Err +} + +// expandPackages takes a list of package or package expressions and invoke go +// list to expand them to packages. In particular, it handles things like "..." +// and ".". +func expandPackages(gopath string, pkgs []string) ([]string, error) { + args := []string{"list"} + args = append(args, pkgs...) + cmd := exec.Command("go", args...) + cmd.Env = fixEnv(gopath) + out, err := cmd.CombinedOutput() + if err != nil { + output := string(out) + if strings.Contains(output, "cannot find package") || + strings.Contains(output, "no buildable Go source files") { + return nil, &MissingError{Err: output} + } + return nil, fmt.Errorf("'go %s' failed with:\n%s", + strings.Join(args, " "), output) + } + names := []string{} + for _, s := range strings.Split(string(out), "\n") { + s = strings.TrimSpace(s) + if s != "" { + names = append(names, s) + } + } + return names, nil +} + +func listPackagesAndDeps(gopath string, pkgs []string) ([]string, error) { + pkgs, err := expandPackages(gopath, pkgs) + if err != nil { + return nil, err + } + args := []string{"list", "-f", "{{range .Deps}}{{.}}|{{end}}"} + args = append(args, pkgs...) + cmd := exec.Command("go", args...) + cmd.Env = fixEnv(gopath) + out, err := cmd.CombinedOutput() + if err != nil { + output := string(out) + if strings.Contains(output, "cannot find package") || + strings.Contains(output, "no buildable Go source files") { + return nil, &MissingError{Err: output} + } + return nil, fmt.Errorf("'go %s' failed with:\n%s", + strings.Join(args, " "), output) + } + deps := []string{} + seen := map[string]bool{} + for _, s := range strings.Split(string(out), "|") { + s = strings.TrimSpace(s) + if s != "" && !seen[s] { + deps = append(deps, s) + seen[s] = true + } + } + for _, pkg := range pkgs { + if !seen[pkg] { + seen[pkg] = true + deps = append(deps, pkg) + } + } + sort.Strings(deps) + return deps, nil +} + +func listStandardPackages(gopath string) ([]string, error) { + return expandPackages(gopath, []string{"std", "cmd"}) +} + +// PkgError reports on missing packages +type PkgError struct { + Err string +} + +// PkgInfo holds identifying package info +type PkgInfo struct { + Name string + Dir string + Root string + ImportPath string + Error *PkgError +} + +func getPackagesInfo(gopath string, pkgs []string) ([]*PkgInfo, error) { + args := []string{"list", "-e", "-json"} + // TODO: split the list for platforms which do not support massive argument + // lists. + args = append(args, pkgs...) + cmd := exec.Command("go", args...) + cmd.Env = fixEnv(gopath) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("go %s failed with:\n%s", + strings.Join(args, " "), string(out)) + } + infos := make([]*PkgInfo, 0, len(pkgs)) + decoder := json.NewDecoder(bytes.NewBuffer(out)) + var derr error + for _, pkg := range pkgs { + info := &PkgInfo{} + derr = decoder.Decode(info) + if derr != nil { + return nil, fmt.Errorf("could not retrieve package information for %s", pkg) + } + if pkg != info.ImportPath { + return nil, fmt.Errorf("package information mismatch: asked for %s, got %s", + pkg, info.ImportPath) + } + if info.Error != nil && info.Name == "" { + info.Name = info.ImportPath + } + infos = append(infos, info) + } + return infos, err +} + +var ( + reLicense = regexp.MustCompile(`(?i)^(?:` + + `((?:un)?licen[sc]e(?:\.[^.]+)?)|` + + `(copy(?:ing|right)(?:\.[^.]+)?)|` + + `)$`) +) + +// scoreLicenseName returns a factor between 0 and 1 weighting how likely +// supplied filename is a license file. +func scoreLicenseName(name string) int8 { + m := reLicense.FindStringSubmatch(name) + switch { + case m == nil: + break + case m[1] != "" || m[2] != "": + return 1 + } + return 0 +} + +// findLicenses looks for license files in package import path, and down to +// parent directories until a file is found or $GOPATH/src is reached. It +// returns a slice of paths all viable files, or a slice containing one empty +// string if none were found. +func findLicenses(info *PkgInfo) ([]string, error) { + path := info.ImportPath + for ; path != "."; path = filepath.Dir(path) { + fis, err := ioutil.ReadDir(filepath.Join(info.Root, "src", path)) + if err != nil { + return []string{""}, err + } + allViableNames := make([]string, 0) + for _, fi := range fis { + if !fi.Mode().IsRegular() { + continue + } + score := scoreLicenseName(fi.Name()) + if score == 1 { + allViableNames = append(allViableNames, filepath.Join(path, fi.Name())) + } + } + if len(allViableNames) > 0 { + return allViableNames, nil + } + } + return []string{""}, nil +} + +// GoPackage represents a top-level package, ex. colors/blue +type GoPackage struct { + PackageName string + RawLicenses []*RawLicense + Err string +} + +// RawLicense holds template-matched file data +type RawLicense struct { + Path string + Score float64 + Template *Template + ExtraWords []string + MissingWords []string +} + +func listPackagesWithLicenses(gopath string, pkgs []string) ([]GoPackage, error) { + templates, err := loadTemplates() + if err != nil { + return nil, err + } + deps, err := listPackagesAndDeps(gopath, pkgs) + if err != nil { + if _, ok := err.(*MissingError); ok { + return nil, err + } + return nil, fmt.Errorf("could not list %s dependencies: %s", + strings.Join(pkgs, " "), err) + } + std, err := listStandardPackages(gopath) + if err != nil { + return nil, fmt.Errorf("could not list standard packages: %s", err) + } + stdSet := map[string]bool{} + for _, n := range std { + stdSet[n] = true + } + infos, err := getPackagesInfo(gopath, deps) + if err != nil { + return nil, err + } + + // Cache matched licenses by path. Useful for package with a lot of + // subpackages like bleve. + matched := map[string]MatchResult{} + + gPackages := []GoPackage{} + for _, info := range infos { + if info.Error != nil { + gPackages = append(gPackages, GoPackage{ + PackageName: info.Name, + Err: info.Error.Err, + RawLicenses: []*RawLicense{{Path: ""}}, + }) + continue + } + if stdSet[info.ImportPath] { + continue + } + paths, err := findLicenses(info) + if err != nil { + return nil, err + } + rawLicenseInfos := []*RawLicense{} + gPackage := GoPackage{PackageName: info.ImportPath} + for _, path := range paths { + rl := RawLicense{Path: path} + if path != "" { + fpath := filepath.Join(info.Root, "src", path) + m, ok := matched[fpath] + if !ok { + data, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + m = matchTemplates(data, templates) + matched[fpath] = m + } + rl.Score = m.Score + rl.Template = m.Template + rl.ExtraWords = m.ExtraWords + rl.MissingWords = m.MissingWords + } + rawLicenseInfos = append(rawLicenseInfos, &rl) + } + gPackage.RawLicenses = rawLicenseInfos + gPackages = append(gPackages, gPackage) + } + return gPackages, nil +} + +// longestCommonPrefix returns the longest common prefix over import path +// components of supplied licenses. +func longestCommonPrefix(gPackages []GoPackage) string { + type Node struct { + Name string + Children map[string]*Node + Shared int + } + // Build a prefix tree. Not super efficient, but easy to do. + root := &Node{ + Children: map[string]*Node{}, + Shared: len(gPackages), + } + for _, l := range gPackages { + n := root + for _, part := range strings.Split(l.PackageName, "/") { + c := n.Children[part] + if c == nil { + c = &Node{ + Name: part, + Children: map[string]*Node{}, + } + n.Children[part] = c + } + c.Shared++ + n = c + } + } + n := root + prefix := []string{} + for { + if len(n.Children) != 1 { + break + } + for _, c := range n.Children { + if c.Shared == len(gPackages) { + // Handle case where there are subpackages: + // prometheus/procfs + // prometheus/procfs/xfs + prefix = append(prefix, c.Name) + } + n = c + break + } + } + return strings.Join(prefix, "/") +} + +// groupPackagesByLicense returns the input packages after grouping them by license +// path and find their longest import path common prefix. Entries with empty +// paths are left unchanged. +func groupPackagesByLicense(gPackages []GoPackage) ([]GoPackage, error) { + paths := map[string][]GoPackage{} + for _, gp := range gPackages { + for _, rl := range gp.RawLicenses { + if rl.Path == "" { + continue + } + paths[rl.Path] = append(paths[rl.Path], gp) + } + } + for k, v := range paths { + if len(v) <= 1 { + continue + } + prefix := longestCommonPrefix(v) + if prefix == "" { + return nil, fmt.Errorf( + "packages share the same license but not common prefix: %v", v) + } + gp := v[0] + gp.PackageName = prefix + paths[k] = []GoPackage{gp} + } + kept := []GoPackage{} + // Ensures only one package with multiple licenses is appended to the list of + // kept packages + seen := make(map[string]bool) + for _, gp := range gPackages { + if len(gp.RawLicenses) == 0 { + kept = append(kept, gp) + continue + } + for _, rl := range gp.RawLicenses { + if rl.Path == "" { + kept = append(kept, gp) + continue + } + if v, ok := paths[rl.Path]; ok { + if _, ok := seen[v[0].PackageName]; !ok { + kept = append(kept, v[0]) + delete(paths, rl.Path) + seen[v[0].PackageName] = true + } + } + } + } + return kept, nil +} + +type projectAndLicenses struct { + Project string `json:"project"` + Licenses []license `json:"licenses,omitempty"` + Error string `json:"error,omitempty"` +} + +type license struct { + Type string `json:"type,omitempty"` + Confidence float64 `json:"confidence,omitempty"` +} + +func licensesToProjectAndLicenses(gPackages []GoPackage) (c []projectAndLicenses, e []projectAndLicenses) { + for _, gp := range gPackages { + if gp.Err != "" { + e = append(e, projectAndLicenses{ + Project: removeVendor(gp.PackageName), + Error: gp.Err, + }) + continue + } + nt := 0 + for _, rl := range gp.RawLicenses { + if rl.Template == nil { + nt++ + } + } + if len(gp.RawLicenses) == nt { + e = append(e, projectAndLicenses{ + Project: removeVendor(gp.PackageName), + Error: "No license detected", + }) + continue + } + ls := []license{} + for _, rl := range gp.RawLicenses { + if rl.Template.Title != "" { + ls = append(ls, license{ + Type: rl.Template.Title, + Confidence: rl.Score, + }) + } + } + c = append(c, projectAndLicenses{ + Project: removeVendor(gp.PackageName), + Licenses: ls, + }) + } + return c, e +} + +func removeVendor(s string) string { + v := "/vendor/" + i := strings.Index(s, v) + if i == -1 { + return s + } + return s[i+len(v):] +} + +func truncateFloat(f float64) float64 { + nf := fmt.Sprintf("%.3f", f) + + var err error + f, err = strconv.ParseFloat(nf, 64) + if err != nil { + panic("unexpected parse float error") + } + return f +} + +func pkgsToLicenses(pkgs []string, overrides string) (pls []projectAndLicenses, ne []projectAndLicenses) { + fplm := make(map[string][]string) + if err := json.Unmarshal([]byte(overrides), &pls); err != nil { + log.Fatal(err) + } + for _, pl := range pls { + for _, l := range pl.Licenses { + fplm[pl.Project] = append(fplm[pl.Project], l.Type) + } + } + + licenses, err := listPackagesWithLicenses("", pkgs) + if err != nil { + log.Fatal(err) + } + if licenses, err = groupPackagesByLicense(licenses); err != nil { + log.Fatal(err) + } + c, e := licensesToProjectAndLicenses(licenses) + + // detected licenses + pls = nil + ls := []license{} + for _, pl := range c { + if fl, ok := fplm[pl.Project]; ok { + for _, l := range fl { + ls = append(ls, license{ + Type: l, + Confidence: 1.0, + }) + } + pl = projectAndLicenses{ + Project: pl.Project, + Licenses: ls, + } + delete(fplm, pl.Project) + } + pls = append(pls, pl) + } + // force add undetected licenses given by overrides + ls = nil + for proj, fl := range fplm { + for _, l := range fl { + ls = append(ls, license{ + Type: l, + Confidence: 1.0, + }) + } + pls = append(pls, projectAndLicenses{ + Project: proj, + Licenses: ls, + }) + } + // missing / error license + for _, pl := range e { + if _, ok := fplm[pl.Project]; !ok { + ne = append(ne, pl) + } + } + + sort.Slice(pls, func(i, j int) bool { return pls[i].Project < pls[j].Project }) + sort.Slice(ne, func(i, j int) bool { return ne[i].Project < ne[j].Project }) + return pls, ne +} + +func main() { + of := flag.String("override-file", "", "a file to overwrite licenses") + flag.Parse() + if flag.NArg() < 1 { + log.Fatal("expect at least one package argument") + } + + overrides := "[]" + if len(*of) != 0 { + b, err := ioutil.ReadFile(*of) + if err != nil { + log.Fatal(err) + } + overrides = string(b) + } + + c, ne := pkgsToLicenses(flag.Args(), overrides) + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + log.Fatal(err) + } + fmt.Println(string(b)) + + if len(ne) != 0 { + fmt.Println("") + b, err := json.MarshalIndent(ne, "", " ") + if err != nil { + log.Fatal(err) + } + fmt.Println(string(b)) + os.Exit(1) + } +}