diff --git a/pkg/db/session/internal/boltdb/boltdb.go b/pkg/db/session/internal/boltdb/boltdb.go index 9476ff1..8d63170 100644 --- a/pkg/db/session/internal/boltdb/boltdb.go +++ b/pkg/db/session/internal/boltdb/boltdb.go @@ -12,6 +12,9 @@ import ( "github.com/pkg/errors" bolt "go.etcd.io/bbolt" + attackTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/attack" + capecTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/capec" + cweTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/cwe" dataTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data" advisoryTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory" advisoryContentTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory/content" @@ -45,6 +48,12 @@ const ( // boltdb: microsoft:kb: -> microsoftkbTypes.KB +// boltdb: attack: -> attackTypes.Attack + +// boltdb: capec: -> capecTypes.Capec + +// boltdb: cwe: -> cweTypes.CWE + // boltdb: datasource: -> datasourceTypes.DataSource type Config struct { @@ -138,102 +147,39 @@ func putMetadata(tx *bolt.Tx, metadata dbTypes.Metadata) error { func (c *Connection) Put(root string) error { if err := c.conn.Update(func(tx *bolt.Tx) error { - if err := func() error { - dataDir := filepath.Join(root, "data") - if _, err := os.Stat(dataDir); err != nil { - if os.IsNotExist(err) { - return nil + for _, spec := range []struct { + dir string + put func(*bolt.Tx, string) error + }{ + {dir: "data", put: putData}, + {dir: "microsoftkb", put: putMicrosoftKB}, + {dir: "attack", put: putAttack}, + {dir: "capec", put: putCAPEC}, + {dir: "cwe", put: putCWE}, + } { + if err := func() error { + d := filepath.Join(root, spec.dir) + if _, err := os.Stat(d); err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "stat %s", d) } - return errors.Wrapf(err, "stat %s", dataDir) - } - - if err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err + if err := filepath.WalkDir(d, func(path string, de fs.DirEntry, err error) error { + if err != nil { + return err + } + if de.IsDir() || filepath.Ext(path) != ".json" { + return nil + } + return spec.put(tx, path) + }); err != nil { + return errors.Wrapf(err, "walk %s", d) } - - if d.IsDir() || filepath.Ext(path) != ".json" { - return nil - } - - f, err := os.Open(path) - if err != nil { - return errors.Wrapf(err, "open %s", path) - } - defer f.Close() - - var data dataTypes.Data - if err := json.UnmarshalRead(f, &data); err != nil { - return errors.Wrapf(err, "unmarshal %s", path) - } - - if err := putDetection(tx, data); err != nil { - return errors.Wrap(err, "put detection") - } - - if err := putAdvisory(tx, data); err != nil { - return errors.Wrap(err, "put advisory") - } - - if err := putVulnerability(tx, data); err != nil { - return errors.Wrap(err, "put vulnerability") - } - - if err := putRoot(tx, data); err != nil { - return errors.Wrap(err, "put root") - } - return nil - }); err != nil { - return errors.Wrapf(err, "walk %s", dataDir) + }(); err != nil { + return errors.Wrapf(err, "put %s data", spec.dir) } - - return nil - }(); err != nil { - return errors.Wrap(err, "put vulnerability data") - } - - if err := func() error { - microsoftkbDir := filepath.Join(root, "microsoftkb") - if _, err := os.Stat(microsoftkbDir); err != nil { - if os.IsNotExist(err) { - return nil - } - return errors.Wrapf(err, "stat %s", microsoftkbDir) - } - - if err := filepath.WalkDir(microsoftkbDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - if d.IsDir() || filepath.Ext(path) != ".json" { - return nil - } - - f, err := os.Open(path) - if err != nil { - return errors.Wrapf(err, "open %s", path) - } - defer f.Close() - - var kb microsoftkbTypes.KB - if err := json.UnmarshalRead(f, &kb); err != nil { - return errors.Wrapf(err, "unmarshal %s", path) - } - - if err := putMicrosoftKB(tx, kb); err != nil { - return errors.Wrap(err, "put microsoft kb") - } - - return nil - }); err != nil { - return errors.Wrapf(err, "walk %s", microsoftkbDir) - } - - return nil - }(); err != nil { - return errors.Wrap(err, "put microsoft kb data") } if err := func() error { @@ -490,29 +436,45 @@ func putRoot(tx *bolt.Tx, data dataTypes.Data) error { return nil } -func putDataSource(tx *bolt.Tx, datasource datasourceTypes.DataSource) error { - sb := tx.Bucket([]byte("datasource")) - if sb == nil { - return errors.Errorf("%q is not exists", "datasource") +func putData(tx *bolt.Tx, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open %s", path) } + defer f.Close() - if sb.Get([]byte(datasource.ID)) != nil { - return errors.Errorf("%q already exists", fmt.Sprintf("datasource -> %s", datasource.ID)) + var data dataTypes.Data + if err := json.UnmarshalRead(f, &data); err != nil { + return errors.Wrapf(err, "unmarshal %s", path) } - bs, err := util.Marshal(datasource) - if err != nil { - return errors.Wrap(err, "marshal datasource") + if err := putDetection(tx, data); err != nil { + return errors.Wrap(err, "put detection") } - - if err := sb.Put([]byte(datasource.ID), bs); err != nil { - return errors.Wrapf(err, "put %q", fmt.Sprintf("datasource -> %q", datasource.ID)) + if err := putAdvisory(tx, data); err != nil { + return errors.Wrap(err, "put advisory") + } + if err := putVulnerability(tx, data); err != nil { + return errors.Wrap(err, "put vulnerability") + } + if err := putRoot(tx, data); err != nil { + return errors.Wrap(err, "put root") } - return nil } -func putMicrosoftKB(tx *bolt.Tx, kb microsoftkbTypes.KB) error { +func putMicrosoftKB(tx *bolt.Tx, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open %s", path) + } + defer f.Close() + + var kb microsoftkbTypes.KB + if err := json.UnmarshalRead(f, &kb); err != nil { + return errors.Wrapf(err, "unmarshal %s", path) + } + eb, err := tx.CreateBucketIfNotExists([]byte(ecosystemTypes.EcosystemTypeMicrosoft)) if err != nil { return errors.Wrapf(err, "create %q if not exists", ecosystemTypes.EcosystemTypeMicrosoft) @@ -543,6 +505,109 @@ func putMicrosoftKB(tx *bolt.Tx, kb microsoftkbTypes.KB) error { return nil } +func putAttack(tx *bolt.Tx, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open %s", path) + } + defer f.Close() + var a attackTypes.Attack + if err := json.UnmarshalRead(f, &a); err != nil { + return errors.Wrapf(err, "unmarshal %s", path) + } + if a.ID == "" { + return nil + } + b, err := tx.CreateBucketIfNotExists([]byte("attack")) + if err != nil { + return errors.Wrapf(err, "create %q bucket", "attack") + } + bs, err := util.Marshal(a) + if err != nil { + return errors.Wrapf(err, "marshal attack %s", a.ID) + } + if err := b.Put([]byte(a.ID), bs); err != nil { + return errors.Wrapf(err, "put attack %s", a.ID) + } + return nil +} + +func putCAPEC(tx *bolt.Tx, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open %s", path) + } + defer f.Close() + var c capecTypes.CAPEC + if err := json.UnmarshalRead(f, &c); err != nil { + return errors.Wrapf(err, "unmarshal %s", path) + } + if c.ID == "" { + return nil + } + b, err := tx.CreateBucketIfNotExists([]byte("capec")) + if err != nil { + return errors.Wrapf(err, "create %q bucket", "capec") + } + bs, err := util.Marshal(c) + if err != nil { + return errors.Wrapf(err, "marshal capec %s", c.ID) + } + if err := b.Put([]byte(c.ID), bs); err != nil { + return errors.Wrapf(err, "put capec %s", c.ID) + } + return nil +} + +func putCWE(tx *bolt.Tx, path string) error { + f, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "open %s", path) + } + defer f.Close() + var w cweTypes.CWE + if err := json.UnmarshalRead(f, &w); err != nil { + return errors.Wrapf(err, "unmarshal %s", path) + } + if w.ID == "" { + return nil + } + b, err := tx.CreateBucketIfNotExists([]byte("cwe")) + if err != nil { + return errors.Wrapf(err, "create %q bucket", "cwe") + } + bs, err := util.Marshal(w) + if err != nil { + return errors.Wrapf(err, "marshal cwe %s", w.ID) + } + if err := b.Put([]byte(w.ID), bs); err != nil { + return errors.Wrapf(err, "put cwe %s", w.ID) + } + return nil +} + +func putDataSource(tx *bolt.Tx, datasource datasourceTypes.DataSource) error { + sb := tx.Bucket([]byte("datasource")) + if sb == nil { + return errors.Errorf("%q is not exists", "datasource") + } + + if sb.Get([]byte(datasource.ID)) != nil { + return errors.Errorf("%q already exists", fmt.Sprintf("datasource -> %s", datasource.ID)) + } + + bs, err := util.Marshal(datasource) + if err != nil { + return errors.Wrap(err, "marshal datasource") + } + + if err := sb.Put([]byte(datasource.ID), bs); err != nil { + return errors.Wrapf(err, "put %q", fmt.Sprintf("datasource -> %q", datasource.ID)) + } + + return nil +} + func (c *Connection) GetRoot(id dataTypes.RootID) (dbTypes.VulnerabilityData, error) { var d dbTypes.VulnerabilityData if err := c.conn.View(func(tx *bolt.Tx) error { @@ -767,6 +832,87 @@ func (c *Connection) GetMicrosoftKB(kbid string) (map[sourceTypes.SourceID]micro return m, nil } +func (c *Connection) GetAttack(id string) (*attackTypes.Attack, error) { + if id == "" { + return nil, nil + } + var a attackTypes.Attack + if err := c.conn.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("attack")) + if b == nil { + return errors.Wrapf(dbTypes.ErrNotFoundAttack, "%q not found", fmt.Sprintf("attack -> %s", id)) + } + bs := b.Get([]byte(id)) + if len(bs) == 0 { + return errors.Wrapf(dbTypes.ErrNotFoundAttack, "%q not found", fmt.Sprintf("attack -> %s", id)) + } + if err := util.Unmarshal(bs, &a); err != nil { + return errors.Wrapf(err, "unmarshal %q", fmt.Sprintf("attack -> %s", id)) + } + return nil + }); err != nil { + if errors.Is(err, dbTypes.ErrNotFoundAttack) { + return nil, err + } + return nil, errors.WithStack(err) + } + return &a, nil +} + +func (c *Connection) GetCAPEC(id string) (*capecTypes.CAPEC, error) { + if id == "" { + return nil, nil + } + var ce capecTypes.CAPEC + if err := c.conn.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("capec")) + if b == nil { + return errors.Wrapf(dbTypes.ErrNotFoundCAPEC, "%q not found", fmt.Sprintf("capec -> %s", id)) + } + bs := b.Get([]byte(id)) + if len(bs) == 0 { + return errors.Wrapf(dbTypes.ErrNotFoundCAPEC, "%q not found", fmt.Sprintf("capec -> %s", id)) + } + if err := util.Unmarshal(bs, &ce); err != nil { + return errors.Wrapf(err, "unmarshal %q", fmt.Sprintf("capec -> %s", id)) + } + return nil + }); err != nil { + if errors.Is(err, dbTypes.ErrNotFoundCAPEC) { + return nil, err + } + return nil, errors.WithStack(err) + } + return &ce, nil +} + +func (c *Connection) GetCWE(id string) (*cweTypes.CWE, error) { + if id == "" { + return nil, nil + } + var w cweTypes.CWE + if err := c.conn.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("cwe")) + if b == nil { + return errors.Wrapf(dbTypes.ErrNotFoundCWE, "%q not found", fmt.Sprintf("cwe -> %s", id)) + } + bs := b.Get([]byte(id)) + if len(bs) == 0 { + return errors.Wrapf(dbTypes.ErrNotFoundCWE, "%q not found", fmt.Sprintf("cwe -> %s", id)) + } + if err := util.Unmarshal(bs, &w); err != nil { + return errors.Wrapf(err, "unmarshal %q", fmt.Sprintf("cwe -> %s", id)) + } + return nil + }); err != nil { + if errors.Is(err, dbTypes.ErrNotFoundCWE) { + return nil, err + } + return nil, errors.WithStack(err) + } + return &w, nil +} + func (c *Connection) GetDataSources() ([]datasourceTypes.DataSource, error) { var ds []datasourceTypes.DataSource if err := c.conn.View(func(tx *bolt.Tx) error { diff --git a/pkg/db/session/internal/rdb/rdb.go b/pkg/db/session/internal/rdb/rdb.go index f78330c..337118a 100644 --- a/pkg/db/session/internal/rdb/rdb.go +++ b/pkg/db/session/internal/rdb/rdb.go @@ -7,6 +7,9 @@ import ( "gorm.io/driver/postgres" "gorm.io/gorm" + attackTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/attack" + capecTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/capec" + cweTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/cwe" dataTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data" advisoryTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory" advisoryContentTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory/content" @@ -120,6 +123,18 @@ func (c *Connection) GetMicrosoftKB(kbid string) (map[sourceTypes.SourceID]micro return nil, errors.New("not implemented yet") } +func (c *Connection) GetAttack(id string) (*attackTypes.Attack, error) { + return nil, errors.New("not implemented yet") +} + +func (c *Connection) GetCAPEC(id string) (*capecTypes.CAPEC, error) { + return nil, errors.New("not implemented yet") +} + +func (c *Connection) GetCWE(id string) (*cweTypes.CWE, error) { + return nil, errors.New("not implemented yet") +} + func (c *Connection) GetDataSources() ([]datasourceTypes.DataSource, error) { return nil, errors.New("not implemented yet") } diff --git a/pkg/db/session/internal/redis/redis.go b/pkg/db/session/internal/redis/redis.go index 29609f3..e4794f6 100644 --- a/pkg/db/session/internal/redis/redis.go +++ b/pkg/db/session/internal/redis/redis.go @@ -4,6 +4,9 @@ import ( "github.com/pkg/errors" "github.com/redis/rueidis" + attackTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/attack" + capecTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/capec" + cweTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/cwe" dataTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data" advisoryTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory" advisoryContentTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory/content" @@ -88,6 +91,18 @@ func (c *Connection) GetMicrosoftKB(kbid string) (map[sourceTypes.SourceID]micro return nil, errors.New("not implemented yet") } +func (c *Connection) GetAttack(id string) (*attackTypes.Attack, error) { + return nil, errors.New("not implemented yet") +} + +func (c *Connection) GetCAPEC(id string) (*capecTypes.CAPEC, error) { + return nil, errors.New("not implemented yet") +} + +func (c *Connection) GetCWE(id string) (*cweTypes.CWE, error) { + return nil, errors.New("not implemented yet") +} + func (c *Connection) GetDataSources() ([]datasourceTypes.DataSource, error) { return nil, errors.New("not implemented yet") } diff --git a/pkg/db/session/session.go b/pkg/db/session/session.go index d8a894e..adc7edc 100644 --- a/pkg/db/session/session.go +++ b/pkg/db/session/session.go @@ -10,6 +10,9 @@ import ( bolt "go.etcd.io/bbolt" "gorm.io/gorm" + attackTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/attack" + capecTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/capec" + cweTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/cwe" dataTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data" advisoryTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory" advisoryContentTypes "github.com/MaineK00n/vuls-data-update/pkg/extract/types/data/advisory/content" @@ -42,6 +45,9 @@ type Storage interface { GetIndex(ecosystemTypes.Ecosystem, string) ([]dataTypes.RootID, error) GetDetection(ecosystemTypes.Ecosystem, dataTypes.RootID) (map[sourceTypes.SourceID][]conditionTypes.Condition, error) GetMicrosoftKB(string) (map[sourceTypes.SourceID]microsoftkbTypes.KB, error) + GetAttack(string) (*attackTypes.Attack, error) + GetCAPEC(string) (*capecTypes.CAPEC, error) + GetCWE(string) (*cweTypes.CWE, error) GetDataSources() ([]datasourceTypes.DataSource, error) GetDataSource(sourceTypes.SourceID) (datasourceTypes.DataSource, error) diff --git a/pkg/db/session/types/types.go b/pkg/db/session/types/types.go index 41f942b..6147418 100644 --- a/pkg/db/session/types/types.go +++ b/pkg/db/session/types/types.go @@ -28,6 +28,9 @@ var ( ErrNotFoundDetection = errors.New("detection not found") ErrNotFoundMicrosoftKB = errors.New("microsoft kb not found") ErrNotFoundDataSource = errors.New("datasource not found") + ErrNotFoundAttack = errors.New("attack not found") + ErrNotFoundCAPEC = errors.New("capec not found") + ErrNotFoundCWE = errors.New("cwe not found") ) type Metadata struct {