pax_global_header00006660000000000000000000000064133756433410014523gustar00rootroot0000000000000052 comment=b083bedef97055c27d1f039428ab183eca7c6450 go-cve-dictionary-0.3.1/000077500000000000000000000000001337564334100150475ustar00rootroot00000000000000go-cve-dictionary-0.3.1/.dockerignore000066400000000000000000000000551337564334100175230ustar00rootroot00000000000000.dockerignore Dockerfile vendor/ cve.sqlite3 go-cve-dictionary-0.3.1/.gitignore000066400000000000000000000001411337564334100170330ustar00rootroot00000000000000.vscode coverage.out vendor/ go-cve-dictionary *.sqlite3 *.sqlite3-shm *.sqlite3-wal tags /dist/ go-cve-dictionary-0.3.1/.goreleaser.yml000066400000000000000000000007311337564334100200010ustar00rootroot00000000000000project_name: go-cve-dictionary release: github: owner: kotakanbe name: go-cve-dictionary builds: - goos: - linux goarch: - amd64 main: . ldflags: -s -w -X main.version={{.Version}} -X main.revision={{.Commit}} binary: go-cve-dictionary archive: format: tar.gz name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' files: - LICENSE - README* snapshot: name_template: SNAPSHOT-{{ .Commit }} go-cve-dictionary-0.3.1/.travis.yml000066400000000000000000000001641337564334100171610ustar00rootroot00000000000000language: go go: - "1.10" after_success: - test -n "$TRAVIS_TAG" && curl -sL https://git.io/goreleaser | bash go-cve-dictionary-0.3.1/Dockerfile000066400000000000000000000011261337564334100170410ustar00rootroot00000000000000FROM golang:alpine as builder RUN apk add --no-cache \ git \ make \ gcc \ musl-dev ENV REPOSITORY github.com/kotakanbe/go-cve-dictionary COPY . $GOPATH/src/$REPOSITORY RUN cd $GOPATH/src/$REPOSITORY && make install FROM alpine:3.7 MAINTAINER hikachan sadayuki-matsuno ENV LOGDIR /var/log/vuls ENV WORKDIR /vuls RUN apk add --no-cache ca-certificates \ && mkdir -p $WORKDIR $LOGDIR COPY --from=builder /go/bin/go-cve-dictionary /usr/local/bin/ VOLUME [$WORKDIR, $LOGDIR] WORKDIR $WORKDIR ENV PWD $WORKDIR ENTRYPOINT ["go-cve-dictionary"] CMD ["--help"] go-cve-dictionary-0.3.1/GNUmakefile000066400000000000000000000024751337564334100171310ustar00rootroot00000000000000.PHONY: \ dep \ depup \ build \ install \ all \ vendor \ lint \ vet \ fmt \ fmtcheck \ pretest \ test \ integration \ cov \ clean SRCS = $(shell git ls-files '*.go') PKGS = ./commands ./config ./db ./log ./models ./fetcher ./fetcher/jvn/xml ./fetcher/nvd ./fetcher/nvd/xml ./fetcher/nvd/json ./server ./util VERSION := $(shell git describe --tags --abbrev=0) REVISION := $(shell git rev-parse --short HEAD) LDFLAGS := -X 'main.version=$(VERSION)' \ -X 'main.revision=$(REVISION)' all: dep build test dep: go get -u github.com/golang/dep/... dep ensure -v depup: go get -u github.com/golang/dep/... dep ensure -update -v build: main.go dep go build -ldflags "$(LDFLAGS)" -o go-cve-dictionary $< install: main.go dep go install -ldflags "$(LDFLAGS)" all: test lint: @ go get -v golang.org/x/lint/golint $(foreach file,$(SRCS),golint $(file) || exit;) vet: $(foreach pkg,$(PKGS),go vet $(pkg);) fmt: gofmt -w $(SRCS) fmtcheck: $(foreach file,$(SRCS),gofmt -d $(file);) pretest: lint vet fmtcheck test: pretest $(foreach pkg,$(PKGS),go test -v $(pkg) || exit;) integration: go test -tags docker_integration -run TestIntegration -v cov: @ go get -v github.com/axw/gocov/gocov @ go get golang.org/x/tools/cmd/cover gocov test | gocov report clean: $(foreach pkg,$(PKGS),go clean $(pkg) || exit;) go-cve-dictionary-0.3.1/Gopkg.lock000066400000000000000000000235261337564334100170000ustar00rootroot00000000000000# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. [[projects]] digest = "1:320e7ead93de9fd2b0e59b50fd92a4d50c1f8ab455d96bc2eb083267453a9709" name = "github.com/asaskevich/govalidator" packages = ["."] pruneopts = "UT" revision = "ccb8e960c48f04d6935e72476ae4a51028f9e22f" version = "v9" [[projects]] digest = "1:e04c00d619875ce5fa67180891984a9b1fadcc031af36bcd7a3509cbdad1df15" name = "github.com/cheggaaa/pb" packages = ["."] pruneopts = "UT" revision = "c112833d014c77e8bde723fd0158e3156951639f" version = "v2.0.6" [[projects]] digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55" name = "github.com/dgrijalva/jwt-go" packages = ["."] pruneopts = "UT" revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" version = "v3.2.0" [[projects]] digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" name = "github.com/fatih/color" packages = ["."] pruneopts = "UT" revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" [[projects]] digest = "1:991bb96360eb8db70a40e9c496769fe6a7bbac4047f8376494d9837397078327" name = "github.com/go-redis/redis" packages = [ ".", "internal", "internal/consistenthash", "internal/hashtag", "internal/pool", "internal/proto", "internal/singleflight", "internal/util", ] pruneopts = "UT" revision = "480db94d33e6088e08d628833b6c0705451d24bb" version = "v6.13.2" [[projects]] digest = "1:adea5a94903eb4384abef30f3d878dc9ff6b6b5b0722da25b82e5169216dfb61" name = "github.com/go-sql-driver/mysql" packages = ["."] pruneopts = "UT" revision = "d523deb1b23d913de5bdada721a6071e71283618" version = "v1.4.0" [[projects]] digest = "1:c4a2528ccbcabf90f9f3c464a5fc9e302d592861bbfd0b7135a7de8a943d0406" name = "github.com/go-stack/stack" packages = ["."] pruneopts = "UT" revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" version = "v1.7.0" [[projects]] branch = "master" digest = "1:b264547c40314ec7619d2cf264e2621953843be7242c140efe1e3119f93877f4" name = "github.com/google/subcommands" packages = ["."] pruneopts = "UT" revision = "5bae204cdfb2d92dcc333d56014bae6a2f6c58b1" [[projects]] branch = "master" digest = "1:32c0e96a63bd093eccf37db757fb314be5996f34de93969321c2cbef893a7bd6" name = "github.com/hashicorp/go-version" packages = ["."] pruneopts = "UT" revision = "270f2f71b1ee587f3b609f00f422b76a6b28f348" [[projects]] digest = "1:89180842090b3c38430d0f311f2a514473bb77a29669d111840cfadd2fac0c7a" name = "github.com/htcat/htcat" packages = ["."] pruneopts = "UT" revision = "2e876d1aa131bd5e3a427b9bfacc5db7dc5a553d" version = "v1.0.2" [[projects]] digest = "1:dacb152d24b21683403e82e4e397e973cdc218befff03b0eba8f7fbffcef325c" name = "github.com/inconshreveable/log15" packages = ["."] pruneopts = "UT" revision = "0decfc6c20d9ca0ad143b0e89dcaa20f810b4fb3" version = "v2.13" [[projects]] digest = "1:8fe19266ce82209076d4a81007ff93f40dd349faca4a917aea59d33956bbd4fd" name = "github.com/jinzhu/gorm" packages = [ ".", "dialects/mysql", "dialects/postgres", "dialects/sqlite", ] pruneopts = "UT" revision = "6ed508ec6a4ecb3531899a69cbc746ccf65a4166" version = "v1.9.1" [[projects]] branch = "master" digest = "1:fd97437fbb6b7dce04132cf06775bd258cce305c44add58eb55ca86c6c325160" name = "github.com/jinzhu/inflection" packages = ["."] pruneopts = "UT" revision = "04140366298a54a039076d798123ffa108fff46c" [[projects]] digest = "1:16dd6b893b78a50564cdde1d9f7ea67224dece11bb0886bd882f1dc3dc1d440d" name = "github.com/k0kubun/pp" packages = ["."] pruneopts = "UT" revision = "027a6d1765d673d337e687394dbe780dd64e2a1e" version = "v2.3.0" [[projects]] branch = "master" digest = "1:bdf08c9b41c029c60ba5dc99443a3ce74eedad842cf2adf9c255513f432422e2" name = "github.com/knqyf263/go-cpe" packages = [ "common", "matching", "naming", ] pruneopts = "UT" revision = "659663f6eca2ff32258e282557e7808115ea498a" [[projects]] digest = "1:87f801436ec4799e0e043ab348ff9f3e95b1b3761138158f0950ac67695cc272" name = "github.com/labstack/echo" packages = [ ".", "middleware", ] pruneopts = "UT" revision = "6d227dfea4d2e52cb76856120b3c17f758139b4e" version = "3.3.5" [[projects]] digest = "1:568171fc14a3d819b112c3e219d351ea7b05e8dad7935c4168c6b3373244a686" name = "github.com/labstack/gommon" packages = [ "bytes", "color", "log", "random", ] pruneopts = "UT" revision = "d6898124de917583f5ff5592ef931d1dfe0ddc05" version = "0.2.6" [[projects]] branch = "master" digest = "1:2d370ad7a48d3523291ad53a04b9b1510c182880927f800b46221cc465427800" name = "github.com/lib/pq" packages = [ ".", "hstore", "oid", ] pruneopts = "UT" revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8" [[projects]] digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" name = "github.com/mattn/go-colorable" packages = ["."] pruneopts = "UT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb" name = "github.com/mattn/go-isatty" packages = ["."] pruneopts = "UT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] digest = "1:e2d1d410fb367567c2b53ed9e2d719d3c1f0891397bb2fa49afd747cfbf1e8e4" name = "github.com/mattn/go-runewidth" packages = ["."] pruneopts = "UT" revision = "9e777a8366cce605130a531d2cd6363d07ad7317" version = "v0.0.2" [[projects]] digest = "1:3cafc6a5a1b8269605d9df4c6956d43d8011fc57f266ca6b9d04da6c09dee548" name = "github.com/mattn/go-sqlite3" packages = ["."] pruneopts = "UT" revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4" version = "v1.9.0" [[projects]] branch = "master" digest = "1:4daa045e1e1f3e23f4b07db6880cdf9f259dab65312dfe244a878e6070faaf77" name = "github.com/olekukonko/tablewriter" packages = ["."] pruneopts = "UT" revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279" [[projects]] digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" packages = ["."] pruneopts = "UT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] branch = "master" digest = "1:c468422f334a6b46a19448ad59aaffdfc0a36b08fdcc1c749a0b29b6453d7e59" name = "github.com/valyala/bytebufferpool" packages = ["."] pruneopts = "UT" revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" [[projects]] branch = "master" digest = "1:268b8bce0064e8c057d7b913605459f9a26dcab864c0886a56d196540fbf003f" name = "github.com/valyala/fasttemplate" packages = ["."] pruneopts = "UT" revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" [[projects]] branch = "master" digest = "1:35aff791260b519f5766ebbd85eb439f488bd1af407c785738fb951105b2adbd" name = "golang.org/x/crypto" packages = [ "acme", "acme/autocert", ] pruneopts = "UT" revision = "f027049dab0ad238e394a753dba2d14753473a04" [[projects]] branch = "master" digest = "1:4d7a8265af700258feaff86722049eb5b787240d66dfaf45ff4962f09de6e0be" name = "golang.org/x/sys" packages = ["unix"] pruneopts = "UT" revision = "acbc56fc7007d2a01796d5bde54f39e3b3e95945" [[projects]] digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" name = "google.golang.org/appengine" packages = ["cloudsql"] pruneopts = "UT" revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" version = "v1.1.0" [[projects]] digest = "1:e626376fab8608a972d47e91b3c1bbbddaecaf1d42b82be6dcc52d10a7557893" name = "gopkg.in/VividCortex/ewma.v1" packages = ["."] pruneopts = "UT" revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce" version = "v1.1.1" [[projects]] digest = "1:256938e7d43c73bd5e7bb97dd281d1ebe294b2928403ee1fbec96249915d1150" name = "gopkg.in/cheggaaa/pb.v2" packages = ["termutil"] pruneopts = "UT" revision = "c112833d014c77e8bde723fd0158e3156951639f" version = "v2.0.6" [[projects]] digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" name = "gopkg.in/fatih/color.v1" packages = ["."] pruneopts = "UT" revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" [[projects]] digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" name = "gopkg.in/mattn/go-colorable.v0" packages = ["."] pruneopts = "UT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb" name = "gopkg.in/mattn/go-isatty.v0" packages = ["."] pruneopts = "UT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] digest = "1:e2d1d410fb367567c2b53ed9e2d719d3c1f0891397bb2fa49afd747cfbf1e8e4" name = "gopkg.in/mattn/go-runewidth.v0" packages = ["."] pruneopts = "UT" revision = "9e777a8366cce605130a531d2cd6363d07ad7317" version = "v0.0.2" [solve-meta] analyzer-name = "dep" analyzer-version = 1 input-imports = [ "github.com/asaskevich/govalidator", "github.com/cheggaaa/pb", "github.com/fatih/color", "github.com/go-redis/redis", "github.com/google/subcommands", "github.com/hashicorp/go-version", "github.com/htcat/htcat", "github.com/inconshreveable/log15", "github.com/jinzhu/gorm", "github.com/jinzhu/gorm/dialects/mysql", "github.com/jinzhu/gorm/dialects/postgres", "github.com/jinzhu/gorm/dialects/sqlite", "github.com/k0kubun/pp", "github.com/knqyf263/go-cpe/common", "github.com/knqyf263/go-cpe/matching", "github.com/knqyf263/go-cpe/naming", "github.com/labstack/echo", "github.com/labstack/echo/middleware", "github.com/mattn/go-sqlite3", "github.com/olekukonko/tablewriter", ] solver-name = "gps-cdcl" solver-version = 1 go-cve-dictionary-0.3.1/Gopkg.toml000066400000000000000000000012201337564334100170060ustar00rootroot00000000000000# Gopkg.toml example # # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html # for detailed Gopkg.toml documentation. # # required = ["github.com/user/thing/cmd/thing"] # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] # # [[constraint]] # name = "github.com/user/project" # version = "1.0.0" # # [[constraint]] # name = "github.com/user/project2" # branch = "dev" # source = "github.com/myfork/project2" # # [[override]] # name = "github.com/x/y" # version = "2.4.0" # # [prune] # non-go = false # go-tests = true # unused-packages = true [prune] go-tests = true unused-packages = true go-cve-dictionary-0.3.1/LICENSE000066400000000000000000000261131337564334100160570ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016 kotakanbe Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. go-cve-dictionary-0.3.1/README.md000066400000000000000000000366141337564334100163400ustar00rootroot00000000000000# go-cve-dictionary This is tool to build a local copy of the NVD (National Vulnerabilities Database) [1] and the Japanese JVN [2], which contain security vulnerabilities according to their CVE identifiers [3] including exhaustive information and a risk score. The local copy is generated in sqlite format, and the tool has a server mode for easy querying. [1] https://en.wikipedia.org/wiki/National_Vulnerability_Database [2] https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures [3] http://jvndb.jvn.jp/apis/termsofuse.html ## Installation ### Install requirements go-cve-dictionary requires the following packages. - SQLite3, MySQL, PostgreSQL or Redis - git - gcc - go v1.7.1 or later - https://golang.org/doc/install Here's an example for Amazon EC2 server. ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc $ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz $ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` Put these lines into /etc/profile.d/goenv.sh ```bash export GOPATH=$HOME/go export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin ``` Set the OS environment variable to current shell ```bash $ source /etc/profile.d/goenv.sh ``` ### Deploy go-cve-dictionary To install: ```bash $ mkdir -p $GOPATH/src/github.com/kotakanbe $ cd $GOPATH/src/github.com/kotakanbe $ git clone https://github.com/kotakanbe/go-cve-dictionary.git $ cd go-cve-dictionary $ make install ``` Create a log output directory. You can use another directory on the command line option (-log-dir). ```bash $ sudo mkdir /var/log/vuls $ sudo chown ec2-user /var/log/vuls $ sudo chmod 700 /var/log/vuls ``` Fetch vulnerability data from NVD. It takes about 10 minutes (on AWS). ```bash $ for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -years $i; done ... snip ... $ ls -alh cve.sqlite3 -rw-r--r-- 1 ec2-user ec2-user 7.0M Mar 24 13:20 cve.sqlite3 ``` Now we have vulnerability data. Start go-cve-dictionary as server mode. ```bash $ go-cve-dictionary server [Mar 24 15:21:55] INFO Opening DB. datafile: /home/ec2-user/cve.sqlite3 [Mar 24 15:21:55] INFO Migrating DB [Mar 24 15:21:56] INFO Starting HTTP Sever... [Mar 24 15:21:56] INFO Listening on 127.0.0.1:1323 ``` ### Update go-cve-dictionary If the DB schema was changed, please specify new SQLite3, MySQL, PostgreSQL or Redis DB file. ```bash $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull $ rm -r vendor $ make install ``` Binary files are created under $GOPATH/bin ---- ## Sample data sources ### Hello HeartBleed ```bash $ curl http://127.0.0.1:1323/cves/CVE-2014-0160 | jq "." { "CveID": "CVE-2014-0160", "Nvd": { "Summary": "The (1) TLS and (2) DTLS implementations in OpenSSL 1.0.1 before 1.0.1g do not properly handle Heartbeat Extension packets, which allows remote attackers to obtain sensitive information from process memory via crafted packets that trigger a buffer over-read, as demonstrated by reading private keys, related to d1_both.c and t1_lib.c, aka the Heartbleed bug.", "Score": 5, "AccessVector": "NETWORK", "AccessComplexity": "LOW", "Authentication": "NONE", "ConfidentialityImpact": "PARTIAL", "IntegrityImpact": "NONE", "AvailabilityImpact": "NONE", "Cpes": null, "References": [ { "Source": "CERT", "Link": "http://www.us-cert.gov/ncas/alerts/TA14-098A" }, ...snip... ], "PublishedDate": "2014-04-07T18:55:03.893-04:00", "LastModifiedDate": "2015-10-22T10:19:38.453-04:00" }, "Jvn": { "Title": "OpenSSL の heartbeat 拡張に情報漏えいの脆弱性", "Summary": "OpenSSL の heartbeat 拡張の実装には、情報漏えいの脆弱性が存在します。TLS や DTLS 通信において OpenSSL のコードを実行しているプロセスのメモリ内容が通信相手に漏えいする可能性があります。", "JvnLink": "http://jvndb.jvn.jp/ja/contents/2014/JVNDB-2014-001920.html", "JvnID": "JVNDB-2014-001920", "Score": 5, "Severity": "Medium", "Vector": "(AV:N/AC:L/Au:N/C:P/I:N/A:N)", "References": [ { "Source": "AT-POLICE", "Link": "http://www.npa.go.jp/cyberpolice/detect/pdf/20140410.pdf" }, ...snip... ], "Cpes": null, "PublishedDate": "2014-04-08T16:13:59+09:00", "LastModifiedDate": "2014-04-08T16:13:59+09:00" } } ``` ### Hello Ruby on Rails 4.0.2 ```bash $ curl -v -H "Accept: application/json" -H "Content-type: application/json" -X POST -d '{"name": "cpe:/a:rubyonrails:ruby_on_rails:4.0.2:-"}' http://localhost:1323/cpes | jq "." [ { "CveID": "CVE-2016-0751", "Nvd": { "CveDetailID": 345, "Summary": "actionpack/lib/action_dispatch/http/mime_type.rb in Action Pack in Ruby on Rails before 3.2.22.1, 4.0.x and 4.1.x before 4.1.14.1, 4.2.x before 4.2.5.1, and 5.x before 5.0.0.beta1.1 does not properly restrict use of the MIME type cache, which allows remote attackers to cause a denial of service (memory consumption) via a crafted HTTP Accept header.", "Score": 5, "AccessVector": "NETWORK", "AccessComplexity": "LOW", "Authentication": "NONE", "ConfidentialityImpact": "NONE", "IntegrityImpact": "NONE", "AvailabilityImpact": "PARTIAL", "Cpes": null, "References": [ { "Source": "MLIST", "Link": "https://groups.google.com/forum/message/raw?msg=ruby-security-ann/9oLY_FCzvoc/5CDXbvpYEgAJ" }, { "Source": "MLIST", "Link": "http://www.openwall.com/lists/oss-security/2016/01/25/9" } ], "PublishedDate": "2016-02-15T21:59:05.877-05:00", "LastModifiedDate": "2016-03-18T21:02:43.817-04:00" }, "Jvn": { "Title": "", "Summary": "", "JvnLink": "", "JvnID": "", "Score": 0, "Severity": "", "Vector": "", "References": null, "Cpes": null, "PublishedDate": "0001-01-01T00:00:00Z", "LastModifiedDate": "0001-01-01T00:00:00Z" } }, ... snip ... ] ``` ---- ## Usage go-cve-dictionary has subcommands - list Display a list of fetched vulnerability databases. - fetchnvd Fetch vulnerability data from NVD(English) - fetchjvn Fetch vulnerability data from JVN(Japanese) - server Start HTTP server ### Usage: List subcommands ```bash $ go-cve-dictionary -help Usage: go-cve-dictionary Subcommands: commands list all command names flags describe all known top-level flags help describe subcommands and their syntax Subcommands for fetchjvn: fetchjvn Fetch Vulnerability dictionary from JVN Subcommands for fetchnvd: fetchnvd Fetch Vulnerability dictionary from NVD Subcommands for server: server Start CVE dictionary HTTP server Use "go-cve-dictionary flags" for a list of top-level flags ``` ![screen shot 2018-03-29 at 14 21 56](https://user-images.githubusercontent.com/534611/38073437-fb2a54b2-3365-11e8-88b5-165f5954f0c9.png) ---- ### Usage: Fetch NVD Data ```bash $ go-cve-dictionary fetchnvd -help fetchnvd: fetchnvd [-latest] [-last2y] [-years] 2015 2016 ... [-dbtype=mysql|postgres|sqlite3|redis] [-dbpath=$PWD/cve.sqlite3 or connection string] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] [-quiet] [-xml] [-log-dir=/path/to/log] [-log-json] For the first time, run the blow command to fetch data for entire period. (It takes about 10 minutes) $ for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -years $i; done -dbpath string /path/to/sqlite3 or SQL connection string (default "/Users/kanbe/go/src/github.com/kotakanbe/go-cve-dictionary/cve.sqlite3") -dbtype string Database type to store data in (sqlite3, mysql, postgres or redis supported) (default "sqlite3") -debug debug mode -debug-sql SQL debug mode -http-proxy string http://proxy-url:port (default: empty) -last2y Refresh NVD data in the last two years recent and modified feeds -latest Refresh recent and modified feeds -log-dir string /path/to/log (default "/var/log/vuls") -log-json output log as JSON -quiet quiet mode (no output) -xml Download [XML](https://nvd.nist.gov/vuln/data-feeds#XML_FEED) Vulnerability Feeds. (default [JSON](https://nvd.nist.gov/vuln/data-feeds#JSON_FEED)) -years Refresh NVD data of specific years ``` - Fetch data for entire period. ```bash for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetchnvd -years $i; done ``` - Fetch data in the last 8 days ```bash $ go-cve-dictionary fetchnvd -latest ``` - Fetch data in the last two years ```bash $ go-cve-dictionary fetchnvd -last2y ``` - Fetch data of specific years ```bash $ go-cve-dictionary fetchnvd -years 2002 2003 2016 ``` ---- ### Usage: Fetch JVN Data ```bash $ go-cve-dictionary fetchjvn -h fetchjvn: fetchjvn [-latest] [-last2y] [-years] 1998 1999 ... [-dbpath=$PWD/cve.sqlite3 or connection string] [-dbtype=mysql|postgres|sqlite3|redis] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] [-quiet] [-log-dir=/path/to/log] [-log-json] -dbpath string /path/to/sqlite3 or SQL connection string (default "$PWD/cve.sqlite3") -dbtype string Database type to store data in (sqlite3, mysql, postgres or redis supported) (default "sqlite3") -debug debug mode -debug-sql SQL debug mode -http-proxy string http://proxy-url:port (default: empty) -last2y Refresh JVN data in the last two years. -latest Refresh JVN data for latest. -quiet quiet mode (no output) -log-dir string /path/to/log (default "/var/log/vuls") -log-json output log as JSON -years Refresh JVN data of specific years. ``` - Fetch data for entire period ```bash for i in `seq 1998 $(date +"%Y")`; do go-cve-dictionary fetchjvn -years $i; done ``` - Fetch data in the last two years ```bash $ go-cve-dictionary fetchjvn -last2y ``` - Fetch data of specific years ```bash $ go-cve-dictionary fetchjvn -years 2002 2003 2016 ``` - Fetch data for latest ```bash $ go-cve-dictionary fetchjvn -latest ``` ---- ### Usage: Run HTTP Server ```bash $ go-cve-dictionary server -h server: server [-bind=127.0.0.1] [-port=8000] [-dbpath=$PWD/cve.sqlite3 or connection string] [-dbtype=mysql|postgres|sqlite3|redis] [-debug] [-debug-sql] [-quiet] [-log-dir=/path/to/log] [-log-json] -bind string HTTP server bind to IP address (default: loop back interface) (default "127.0.0.1") -dbpath string /path/to/sqlite3 or SQL connection string (default : $PWD/cve.sqlite3) -dbtype string Database type to store data in (sqlite3, mysql, postgres or redis supported) (default "sqlite3") -debug debug mode (default: false) -debug-sql SQL debug mode (default: false) -quiet quiet mode (no output) -log-dir string /path/to/log (default "/var/log/vuls") -log-json output log as JSON -port string HTTP server port number (default: 1323) (default "1323") ``` ---- ### Usage: Use MySQL as a DB storage back-end - fetchnvd ```bash $ go-cve-dictionary fetchnvd -last2y \ -dbtype mysql \ -dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true" ``` - fetchjvn ```bash $ go-cve-dictionary fetchjvn -last2y \ -dbtype mysql \ -dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true" ``` - server ```bash $ go-cve-dictionary server \ -dbtype mysql \ -dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true" ``` ### Usage: Use Postgres as a DB storage back-end - fetchnvd ```bash $ go-cve-dictionary fetchnvd -last2y \ -dbtype postgres \ -dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password" ``` - fetchjvn ```bash $ go-cve-dictionary fetchjvn -last2y \ -dbtype postgres \ -dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password" ``` - server ```bash $ go-cve-dictionary server \ -dbtype postgres \ -dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password" ``` ### Usage: Use Redis as a DB storage back-end - fetchnvd ```bash $ go-cve-dictionary fetchnvd -last2y \ -dbtype redis \ -dbpath "redis://localhost/0" ``` - fetchjvn ```bash $ go-cve-dictionary fetchjvn -last2y \ -dbtype redis \ -dbpath "redis://localhost/0" ``` - server ```bash $ go-cve-dictionary server \ -dbtype redis \ -dbpath "redis://localhost/0" ``` ---- ## Misc - HTTP Proxy Support If your system at behind HTTP proxy, you have to specify -http-proxy option. - How to daemonize go-cve-dictionary Use Systemd, Upstart or supervisord, daemontools... - How to update vulnerability data automatically Use job scheduler like cron (with -last2y or -latest option). - How to cross compile ```bash $ cd /path/to/your/local-git-repository/go-cve-dictionary $ GOOS=linux GOARCH=amd64 go build -o cvedict.amd64 ``` - Logging go-cve-dictionary writes a log under -log-path specified directory (default is /var/log/vuls/). - Debug Run with -debug, -sql-debug option. ---- ## Data Source - [NVD](https://nvd.nist.gov/) - [JVN(Japanese)](http://jvndb.jvn.jp/apis/myjvn/) ---- ## Authors kotakanbe ([@kotakanbe](https://twitter.com/kotakanbe)) created go-cve-dictionary and [these fine people](https://github.com/future-architect/go-cve-dictionary/graphs/contributors) have contributed. ---- ## How to Contribute 1. fork a repository: github.com/kotakanbe/go-cve-dictionary to github.com/you/repository 1. get original code: github.com/kotakanbe/go-cve-dictionary 1. work on original code 1. add remote to your repository: git remote add myfork https://github.com/you/repo.git 1. push your changes: git push myfork 1. create a new Pull Request - see [GitHub and Go: forking, pull requests, and go-getting](http://blog.campoy.cat/2014/03/github-and-go-forking-pull-requests-and.html) ---- ## Licence Please see [LICENSE](https://github.com/kotakanbe/go-cve-dictionary/blob/master/LICENSE). ---- ## Additional License - [NVD](https://nvd.nist.gov/faq) > How can my organization use the NVD data within our own products and services? > All NVD data is freely available from our XML Data Feeds. There are no fees, licensing restrictions, or even a requirement to register. All NIST publications are available in the public domain according to Title 17 of the United States Code. Acknowledgment of the NVD when using our information is appreciated. In addition, please email nvd@nist.gov to let us know how the information is being used. - [JVN](http://jvndb.jvn.jp/apis/termsofuse.html) go-cve-dictionary-0.3.1/commands/000077500000000000000000000000001337564334100166505ustar00rootroot00000000000000go-cve-dictionary-0.3.1/commands/fetchjvn.go000066400000000000000000000122651337564334100210140ustar00rootroot00000000000000package commands import ( "context" "flag" "os" "strconv" "time" "github.com/google/subcommands" c "github.com/kotakanbe/go-cve-dictionary/config" db "github.com/kotakanbe/go-cve-dictionary/db" jvn "github.com/kotakanbe/go-cve-dictionary/fetcher/jvn/xml" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" util "github.com/kotakanbe/go-cve-dictionary/util" ) // FetchJvnCmd is Subcommand for fetch JVN information. type FetchJvnCmd struct { debug bool debugSQL bool quiet bool logDir string logJSON bool dbpath string dbtype string dumpPath string latest bool last2Y bool years bool httpProxy string } // Name return subcommand name func (*FetchJvnCmd) Name() string { return "fetchjvn" } // Synopsis return synopsis func (*FetchJvnCmd) Synopsis() string { return "Fetch Vulnerability dictionary from JVN" } // Usage return usage func (*FetchJvnCmd) Usage() string { return `fetchjvn: fetchjvn [-latest] [-last2y] [-years] 1998 1999 ... [-dbpath=$PWD/cve.sqlite3 or connection string] [-dbtype=mysql|postgres|sqlite3|redis] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] [-quiet] [-log-dir=/path/to/log] [-log-json] ` } // SetFlags set flag func (p *FetchJvnCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.debug, "debug", false, "debug mode") f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode") f.BoolVar(&p.quiet, "quiet", false, "quiet mode (no output)") defaultLogDir := util.GetDefaultLogDir() f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log") f.BoolVar(&p.logJSON, "log-json", false, "output log as JSON") pwd := os.Getenv("PWD") f.StringVar(&p.dbpath, "dbpath", pwd+"/cve.sqlite3", "/path/to/sqlite3 or SQL connection string") f.StringVar(&p.dbtype, "dbtype", "sqlite3", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") f.BoolVar(&p.latest, "latest", false, "Refresh JVN data for latest.") f.BoolVar(&p.last2Y, "last2y", false, "Refresh JVN data in the last two years.") f.BoolVar(&p.years, "years", false, "Refresh JVN data of specific years.") f.StringVar( &p.httpProxy, "http-proxy", "", "http://proxy-url:port (default: empty)", ) } // Execute execute func (p *FetchJvnCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.Debug = p.debug c.Conf.Quiet = p.quiet c.Conf.DebugSQL = p.debugSQL c.Conf.DBPath = p.dbpath c.Conf.DBType = p.dbtype // c.Conf.DumpPath = p.dumpPath c.Conf.HTTPProxy = p.httpProxy log.SetLogger(p.logDir, c.Conf.Quiet, c.Conf.Debug, p.logJSON) if !c.Conf.Validate() { return subcommands.ExitUsageError } years := []int{} thisYear := time.Now().Year() switch { case p.latest: years = append(years, c.Latest) case p.last2Y: for i := 0; i < 2; i++ { years = append(years, thisYear-i) } years = append(years, c.Latest) case p.years: if len(f.Args()) == 0 { log.Errorf("Specify years to fetch (from 1998 to %d)", thisYear) return subcommands.ExitUsageError } for _, arg := range f.Args() { year, err := strconv.Atoi(arg) if err != nil || year < 1998 || time.Now().Year() < year { log.Errorf("Specify years to fetch (from 1998 to %d), arg: %s", thisYear, arg) return subcommands.ExitUsageError } found := false for _, y := range years { if y == year { found = true break } } if !found { years = append(years, year) } } years = append(years, c.Latest) default: log.Errorf("specify -latest, -last2y or -years") return subcommands.ExitUsageError } driver, locked, err := db.NewDB(c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL) if err != nil { if locked { log.Errorf("Failed to Open DB. Close DB connection before fetching: %s", err) return subcommands.ExitFailure } log.Errorf("%s", err) return subcommands.ExitFailure } defer driver.CloseDB() metas, err := jvn.FetchLatestFeedMeta(driver, years) if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } if len(metas) == 0 { log.Errorf("No meta files fetched") return subcommands.ExitFailure } //TODO use meta.Status() needUpdates := []models.FeedMeta{} for _, m := range metas { if m.Newly() { needUpdates = append(needUpdates, m) log.Infof("Newly : %s", m.URL) } else if m.OutDated() { needUpdates = append(needUpdates, m) log.Infof("Outdated : %s", m.URL) } else { log.Infof("Up to date: %s", m.URL) } } if len(needUpdates) == 0 { log.Infof("Already up to date") return subcommands.ExitSuccess } log.Infof("Fetcling CVE information from JVN.") cves, err := jvn.FetchConvert(needUpdates) if err != nil { log.Errorf("Failed to fetch JVN: %s", err) return subcommands.ExitFailure } log.Infof("Fetched %d CVEs", len(cves)) log.Infof("Inserting JVN into DB (%s).", driver.Name()) if err := driver.InsertJvn(cves); err != nil { log.Fatalf("Failed to insert. dbpath: %s, err: %s", c.Conf.DBPath, err) return subcommands.ExitFailure } if err := jvn.UpdateMeta(driver, needUpdates); err != nil { log.Fatalf("Failed to Update meta. dbpath: %s, err: %s", c.Conf.DBPath, err) return subcommands.ExitFailure } return subcommands.ExitSuccess } go-cve-dictionary-0.3.1/commands/fetchnvd.go000066400000000000000000000137311337564334100210050ustar00rootroot00000000000000package commands import ( "context" "flag" "os" "strconv" "time" "github.com/google/subcommands" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" "github.com/kotakanbe/go-cve-dictionary/fetcher/nvd" nvdjson "github.com/kotakanbe/go-cve-dictionary/fetcher/nvd/json" nvdxml "github.com/kotakanbe/go-cve-dictionary/fetcher/nvd/xml" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" "github.com/kotakanbe/go-cve-dictionary/util" ) // FetchNvdCmd is Subcommand for fetch Nvd information. type FetchNvdCmd struct { debug bool debugSQL bool quiet bool logDir string logJSON bool dbpath string dbtype string Latest bool last2Y bool years bool httpProxy string nvdXML bool } // Name return subcommand name func (*FetchNvdCmd) Name() string { return "fetchnvd" } // Synopsis return synopsis func (*FetchNvdCmd) Synopsis() string { return "Fetch Vulnerability dictionary from NVD" } // Usage return usage func (*FetchNvdCmd) Usage() string { return `fetchnvd: fetchnvd [-latest] [-last2y] [-years] 2015 2016 ... [-dbtype=mysql|postgres|sqlite3|redis] [-dbpath=$PWD/cve.sqlite3 or connection string] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] [-quiet] [-xml] [-log-dir=/path/to/log] [-log-json] For the first time, run the blow command to fetch data for entire period. (It takes about 10 minutes) $ for i in ` + "`seq 2002 $(date +\"%Y\")`;" + ` do go-cve-dictionary fetchnvd -years $i; done ` } // SetFlags set flag func (p *FetchNvdCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.debug, "debug", false, "debug mode") f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode") f.BoolVar(&p.quiet, "quiet", false, "quiet mode (no output)") defaultLogDir := util.GetDefaultLogDir() f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log") f.BoolVar(&p.logJSON, "log-json", false, "output log as JSON") pwd := os.Getenv("PWD") f.StringVar(&p.dbpath, "dbpath", pwd+"/cve.sqlite3", "/path/to/sqlite3 or SQL connection string") f.StringVar(&p.dbtype, "dbtype", "sqlite3", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") f.BoolVar(&p.last2Y, "last2y", false, "Refresh NVD data in the last two years recent and modified feeds") f.BoolVar(&p.Latest, "latest", false, "Refresh recent and modified feeds") f.BoolVar(&p.years, "years", false, "Refresh NVD data of specific years") f.BoolVar(&p.nvdXML, "xml", false, "Download [XML](https://nvd.nist.gov/vuln/data-feeds#XML_FEED) Vulnerability Feeds. (default [JSON](https://nvd.nist.gov/vuln/data-feeds#JSON_FEED))") f.StringVar( &p.httpProxy, "http-proxy", "", "http://proxy-url:port (default: empty)", ) } // Execute execute func (p *FetchNvdCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.Debug = p.debug c.Conf.Quiet = p.quiet log.SetLogger(p.logDir, c.Conf.Quiet, c.Conf.Debug, p.logJSON) c.Conf.DebugSQL = p.debugSQL c.Conf.DBPath = p.dbpath c.Conf.DBType = p.dbtype c.Conf.HTTPProxy = p.httpProxy c.Conf.NVDXML = p.nvdXML if !c.Conf.Validate() { return subcommands.ExitUsageError } years := []int{} thisYear := time.Now().Year() switch { case p.Latest: years = append(years, c.Latest) case p.last2Y: for i := 0; i < 2; i++ { years = append(years, thisYear-i) } years = append(years, c.Latest) case p.years: if len(f.Args()) == 0 { log.Errorf("Specify years to fetch (from 2002 to %d)", thisYear) return subcommands.ExitUsageError } for _, arg := range f.Args() { year, err := strconv.Atoi(arg) if err != nil || year < 2002 || time.Now().Year() < year { log.Errorf("Specify years to fetch (from 2002 to %d), arg: %s", thisYear, arg) return subcommands.ExitUsageError } found := false for _, y := range years { if y == year { found = true break } } if !found { years = append(years, year) } } years = append(years, c.Latest) default: log.Errorf("specify -last2y, -latest or -years") return subcommands.ExitUsageError } driver, locked, err := db.NewDB(c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL) if err != nil { if locked { log.Errorf("Failed to Open DB. Close DB connection before fetching: %s", err) return subcommands.ExitFailure } log.Errorf("%s", err) return subcommands.ExitFailure } defer driver.CloseDB() metas, err := nvd.FetchLatestFeedMeta(driver, years, c.Conf.NVDXML) if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } if len(metas) == 0 { log.Errorf("No meta files fetched") return subcommands.ExitFailure } //TODO use meta.Status() needUpdates := []models.FeedMeta{} for _, m := range metas { if m.Newly() { needUpdates = append(needUpdates, m) log.Infof(" Newly: %s", m.URL) } else if m.OutDated() { needUpdates = append(needUpdates, m) log.Infof(" Outdated: %s", m.URL) } else { log.Infof("Up to date: %s", m.URL) } } if len(needUpdates) == 0 { log.Infof("Already up to date") return subcommands.ExitSuccess } cves := []models.CveDetail{} if c.Conf.NVDXML { cves, err = nvdxml.FetchConvert(needUpdates) } else { cves, err = nvdjson.FetchConvert(needUpdates) } if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } log.Infof("Fetched %d CVEs", len(cves)) log.Infof("Inserting NVD into DB (%s).", driver.Name()) if c.Conf.NVDXML { if err := driver.InsertNvdXML(cves); err != nil { log.Errorf("Failed to insert. dbpath: %s, err: %s", c.Conf.DBPath, err) return subcommands.ExitFailure } } else { if err := driver.InsertNvdJSON(cves); err != nil { log.Errorf("Failed to insert. dbpath: %s, err: %s", c.Conf.DBPath, err) return subcommands.ExitFailure } } if err := nvd.UpdateMeta(driver, needUpdates); err != nil { log.Fatalf("Failed to Update meta. dbpath: %s, err: %s", c.Conf.DBPath, err) return subcommands.ExitFailure } return subcommands.ExitSuccess } go-cve-dictionary-0.3.1/commands/list.go000066400000000000000000000112361337564334100201550ustar00rootroot00000000000000package commands import ( "context" "flag" "fmt" "os" "sort" "github.com/google/subcommands" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" jvn "github.com/kotakanbe/go-cve-dictionary/fetcher/jvn/xml" "github.com/kotakanbe/go-cve-dictionary/fetcher/nvd" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" "github.com/kotakanbe/go-cve-dictionary/util" "github.com/olekukonko/tablewriter" ) // ListCmd is Subcommand for fetch Nvd information. type ListCmd struct { debug bool debugSQL bool logDir string logJSON bool dbpath string dbtype string httpProxy string } // Name return subcommand name func (*ListCmd) Name() string { return "list" } // Synopsis return synopsis func (*ListCmd) Synopsis() string { return "Show a list of fetched feeds" } // Usage return usage func (*ListCmd) Usage() string { return `list: fetchnvd [-dbtype=mysql|postgres|sqlite3|redis] [-dbpath=$PWD/cve.sqlite3 or connection string] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] [-log-dir=/path/to/log] [-log-json] ` } // SetFlags set flag func (p *ListCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.debug, "debug", false, "debug mode") f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode") defaultLogDir := util.GetDefaultLogDir() f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log") f.BoolVar(&p.logJSON, "log-json", false, "output log as JSON") pwd := os.Getenv("PWD") f.StringVar(&p.dbpath, "dbpath", pwd+"/cve.sqlite3", "/path/to/sqlite3 or SQL connection string") f.StringVar(&p.dbtype, "dbtype", "sqlite3", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") f.StringVar( &p.httpProxy, "http-proxy", "", "http://proxy-url:port (default: empty)", ) } // Execute execute func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.Debug = p.debug c.Conf.DebugSQL = p.debugSQL c.Conf.DBPath = p.dbpath c.Conf.DBType = p.dbtype c.Conf.HTTPProxy = p.httpProxy log.SetLogger(p.logDir, c.Conf.Quiet, c.Conf.Debug, p.logJSON) if !c.Conf.Validate() { return subcommands.ExitUsageError } driver, locked, err := db.NewDB(c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL) if err != nil { if locked { log.Errorf("Failed to Open DB. Close DB connection: %s", err) return subcommands.ExitFailure } log.Errorf("%s", err) return subcommands.ExitFailure } defer driver.CloseDB() jsonMetas, xmlMetas, err := nvd.ListFetchedFeeds(driver) if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } sort.Slice(jsonMetas, func(i, j int) bool { return jsonMetas[i].URL < jsonMetas[j].URL }) sort.Slice(xmlMetas, func(i, j int) bool { return xmlMetas[i].URL < xmlMetas[j].URL }) jvnMetas, err := jvn.ListFetchedFeeds(driver) if err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } sort.Slice(jvnMetas, func(i, j int) bool { return jvnMetas[i].URL < jvnMetas[j].URL }) metas := []models.FeedMeta{} for _, mm := range [][]models.FeedMeta{jsonMetas, xmlMetas, jvnMetas} { metas = append(metas, mm...) } data := [][]string{} for _, meta := range metas { data = append(data, meta.ToTableWriterRow()) } table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Source", "Year", "Status", "Fetched", "Latest"}) table.SetBorder(true) table.SetHeaderColor( tablewriter.Colors{tablewriter.Bold}, tablewriter.Colors{tablewriter.Bold}, tablewriter.Colors{tablewriter.Bold}, tablewriter.Colors{tablewriter.Bold}, tablewriter.Colors{tablewriter.Bold}) table.AppendBulk(data) table.Render() cmds := []string{} for _, mm := range [][]models.FeedMeta{jsonMetas, xmlMetas, jvnMetas} { cmd := getUpdateCommand(mm) if cmd != "" { cmds = append(cmds, cmd) } } if 0 < len(cmds) { fmt.Printf("\nTo update feeds, execute the following commands.\n") for _, cmd := range cmds { fmt.Println(cmd) } } return subcommands.ExitSuccess } func getUpdateCommand(metas []models.FeedMeta) string { if len(metas) == 0 { return "" } years := map[string]bool{} latest := false for _, meta := range metas { if meta.OutDated() { y, _, err := meta.Year() if err != nil { log.Errorf("err") continue } switch y { case "modified", "recent": latest = true default: years[y] = true } } } opt := metas[0].FetchOption() if len(years) == 0 && latest { return fmt.Sprintf("$ go-cve-dictionary %s -latest", opt) } if len(years) == 0 { return "" } opt += " -years" for y := range years { opt += " " + y } return fmt.Sprintf("$ go-cve-dictionary %s", opt) } go-cve-dictionary-0.3.1/commands/server.go000066400000000000000000000063441337564334100205140ustar00rootroot00000000000000package commands import ( "context" "flag" "os" "github.com/google/subcommands" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" log "github.com/kotakanbe/go-cve-dictionary/log" server "github.com/kotakanbe/go-cve-dictionary/server" "github.com/kotakanbe/go-cve-dictionary/util" ) // ServerCmd is Subcommand for CVE dictionary HTTP Server type ServerCmd struct { debug bool debugSQL bool quiet bool logDir string logJSON bool dbpath string dbtype string bind string port string } // Name return subcommand name func (*ServerCmd) Name() string { return "server" } // Synopsis return synopsis func (*ServerCmd) Synopsis() string { return "Start CVE dictionary HTTP server" } // Usage return usage func (*ServerCmd) Usage() string { return `server: server [-bind=127.0.0.1] [-port=8000] [-dbpath=$PWD/cve.sqlite3 or connection string] [-dbtype=mysql|postgres|sqlite3|redis] [-debug] [-debug-sql] [-quiet] [-log-dir=/path/to/log] [-log-json] ` } // SetFlags set flag func (p *ServerCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.debug, "debug", false, "debug mode (default: false)") f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode (default: false)") f.BoolVar(&p.quiet, "quiet", false, "quiet mode (no output)") defaultLogDir := util.GetDefaultLogDir() f.StringVar(&p.logDir, "log-dir", defaultLogDir, "/path/to/log") f.BoolVar(&p.logJSON, "log-json", false, "output log as JSON") pwd := os.Getenv("PWD") f.StringVar(&p.dbpath, "dbpath", pwd+"/cve.sqlite3", "/path/to/sqlite3 or SQL connection string") f.StringVar(&p.dbtype, "dbtype", "sqlite3", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") f.StringVar(&p.bind, "bind", "127.0.0.1", "HTTP server bind to IP address (default: loop back interface)") f.StringVar(&p.port, "port", "1323", "HTTP server port number (default: 1323)") } // Execute execute func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.Debug = p.debug c.Conf.Quiet = p.quiet log.SetLogger(p.logDir, c.Conf.Quiet, c.Conf.Debug, p.logJSON) c.Conf.DebugSQL = p.debugSQL c.Conf.Bind = p.bind c.Conf.Port = p.port c.Conf.DBPath = p.dbpath c.Conf.DBType = p.dbtype if !c.Conf.Validate() { return subcommands.ExitUsageError } driver, locked, err := db.NewDB(c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL) if err != nil { if locked { log.Errorf("Failed to Open DB. Close DB connection: %s", err) return subcommands.ExitFailure } log.Errorf("%s", err) return subcommands.ExitFailure } defer driver.CloseDB() var count int if count, err = driver.CountNvd(); err != nil { log.Errorf("Failed to count NVD table: %s", err) return subcommands.ExitFailure } if count == 0 { log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD") log.Infof("") log.Infof(" for i in `seq 2002 $(date +\"%%Y\")`; do go-cve-dictionary fetchnvd -years $i ; done") log.Infof("") return subcommands.ExitSuccess } log.Infof("Starting HTTP Server...") if err = server.Start(p.logDir, driver); err != nil { log.Errorf("%s", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } go-cve-dictionary-0.3.1/config/000077500000000000000000000000001337564334100163145ustar00rootroot00000000000000go-cve-dictionary-0.3.1/config/config.go000066400000000000000000000022031337564334100201050ustar00rootroot00000000000000package config import ( valid "github.com/asaskevich/govalidator" log "github.com/kotakanbe/go-cve-dictionary/log" ) // Latest is the fetch option of fetching modified NVD const Latest = -1 // Conf has Configuration var Conf Config // Config has config type Config struct { Debug bool DebugSQL bool Quiet bool DumpPath string DBPath string DBType string FetchJvnWeek bool FetchJvnMonth bool FetchJvnEntire bool FetchJvnPeriodChar string Bind string `valid:"ipv4"` Port string `valid:"port"` //TODO validate HTTPProxy string NVDXML bool } // Validate validates configuration func (c *Config) Validate() bool { if c.DBType == "sqlite3" { if ok, _ := valid.IsFilePath(c.DBPath); !ok { log.Errorf("SQLite3 DB path must be a *Absolute* file path. dbpath: %s", c.DBPath) return false } } if len(c.DumpPath) != 0 { if ok, _ := valid.IsFilePath(c.DumpPath); !ok { log.Errorf("JSON path must be a *Absolute* file path. dumppath: %s", c.DumpPath) return false } } _, err := valid.ValidateStruct(c) if err != nil { log.Errorf("error: " + err.Error()) return false } return true } go-cve-dictionary-0.3.1/config/config_test.go000066400000000000000000000000171337564334100211450ustar00rootroot00000000000000package config go-cve-dictionary-0.3.1/db/000077500000000000000000000000001337564334100154345ustar00rootroot00000000000000go-cve-dictionary-0.3.1/db/db.go000066400000000000000000000114651337564334100163570ustar00rootroot00000000000000package db import ( "fmt" "strings" version "github.com/hashicorp/go-version" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" "github.com/knqyf263/go-cpe/common" "github.com/knqyf263/go-cpe/matching" "github.com/knqyf263/go-cpe/naming" ) // DB is interface for a database driver type DB interface { Name() string CloseDB() error Get(string) (*models.CveDetail, error) GetMulti([]string) (map[string]models.CveDetail, error) GetByCpeURI(string) ([]models.CveDetail, error) InsertJvn([]models.CveDetail) error InsertNvdXML([]models.CveDetail) error InsertNvdJSON([]models.CveDetail) error CountNvd() (int, error) UpsertFeedHash(models.FeedMeta) error GetFetchedFeedMeta(string) (*models.FeedMeta, error) GetFetchedFeedMetas() ([]models.FeedMeta, error) } // NewDB return DB accessor. func NewDB(dbType, dbpath string, debugSQL bool) (DB, bool, error) { switch dbType { case dialectSqlite3, dialectMysql, dialectPostgreSQL: return NewRDB(dbType, dbpath, debugSQL) case dialectRedis: return NewRedis(dbType, dbpath, debugSQL) } return nil, false, fmt.Errorf("Invalid database dialect, %s", dbType) } func chunkSlice(l []models.CveDetail, n int) chan []models.CveDetail { ch := make(chan []models.CveDetail) go func() { for i := 0; i < len(l); i += n { fromIdx := i toIdx := i + n if toIdx > len(l) { toIdx = len(l) } ch <- l[fromIdx:toIdx] } close(ch) }() return ch } func parseCpeURI(cpe22uri string) (*models.CpeBase, error) { wfn, err := naming.UnbindURI(cpe22uri) if err != nil { return nil, err } return &models.CpeBase{ URI: naming.BindToURI(wfn), FormattedString: naming.BindToFS(wfn), WellFormedName: wfn.String(), CpeWFN: models.CpeWFN{ Part: fmt.Sprintf("%s", wfn.Get(common.AttributePart)), Vendor: fmt.Sprintf("%s", wfn.Get(common.AttributeVendor)), Product: fmt.Sprintf("%s", wfn.Get(common.AttributeProduct)), Version: fmt.Sprintf("%s", wfn.Get(common.AttributeVersion)), Update: fmt.Sprintf("%s", wfn.Get(common.AttributeUpdate)), Edition: fmt.Sprintf("%s", wfn.Get(common.AttributeEdition)), Language: fmt.Sprintf("%s", wfn.Get(common.AttributeLanguage)), SoftwareEdition: fmt.Sprintf("%s", wfn.Get(common.AttributeSwEdition)), TargetSW: fmt.Sprintf("%s", wfn.Get(common.AttributeTargetSw)), TargetHW: fmt.Sprintf("%s", wfn.Get(common.AttributeTargetHw)), Other: fmt.Sprintf("%s", wfn.Get(common.AttributeOther)), }, }, nil } func makeVersionConstraint(dict models.Cpe) string { constraints := []string{} if dict.VersionStartIncluding != "" { constraints = append(constraints, ">= "+dict.VersionStartIncluding) } if dict.VersionStartExcluding != "" { constraints = append(constraints, "> "+dict.VersionStartExcluding) } if dict.VersionEndIncluding != "" { constraints = append(constraints, "<= "+dict.VersionEndIncluding) } if dict.VersionEndExcluding != "" { constraints = append(constraints, "< "+dict.VersionEndExcluding) } return strings.Join(constraints, ", ") } func match(specified common.WellFormedName, cpe models.Cpe) (bool, error) { wfn, err := naming.UnbindURI(cpe.URI) if err != nil { return false, err } if wfn.Get("part") != specified.Get("part") || wfn.Get("vendor") != specified.Get("vendor") || wfn.Get("product") != specified.Get("product") { return false, nil } if matching.IsEqual(specified, wfn) { log.Debugf("%s equals %s", specified.String(), cpe.URI) return true, nil } ver := fmt.Sprintf("%s", specified.Get(common.AttributeVersion)) switch ver { case "NA", "ANY": if matching.IsSuperset(wfn, specified) { log.Debugf("%s is superset of %s", cpe.URI, specified.String()) return true, nil } if matching.IsSubset(wfn, specified) { log.Debugf("%s is subset of %s", cpe.URI, specified.String()) return true, nil } default: ver = strings.Replace(ver, `\`, "", -1) v, err := version.NewVersion(ver) if err != nil { return false, err } constraintStr := makeVersionConstraint(cpe) if constraintStr != "" { constraints, err := version.NewConstraint(constraintStr) if err != nil { return false, err } if constraints.Check(v) { log.Debugf("%s satisfies constraints %s", v, constraintStr) return true, nil } } } return false, nil } func matchCpe(specified common.WellFormedName, cve *models.CveDetail) (bool, error) { cpes := []models.Cpe{} if cve.NvdJSON != nil { cpes = append(cpes, cve.NvdJSON.Cpes...) } if cve.NvdXML != nil { cpes = append(cpes, cve.NvdXML.Cpes...) } if cve.Jvn != nil { cpes = append(cpes, cve.Jvn.Cpes...) } for _, cpe := range cpes { match, err := match(specified, cpe) if err != nil { return false, err } if match { return true, nil } } return false, nil } go-cve-dictionary-0.3.1/db/db_test.go000066400000000000000000000135461337564334100174200ustar00rootroot00000000000000package db import ( "os" "testing" "github.com/knqyf263/go-cpe/naming" "github.com/kotakanbe/go-cve-dictionary/models" ) func TestMain(m *testing.M) { // log.Initialize("/tmp", true, os.Stderr) code := m.Run() os.Exit(code) } // move to rdb_test.go func TestMakeVersionConstraint(t *testing.T) { var testdata = []struct { cpe models.Cpe constraint string }{ { cpe: models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Part: "a", Vendor: "cisco", Product: "node-jose", Version: "", Update: "", Edition: "", Language: "", SoftwareEdition: "", TargetSW: "", TargetHW: "", Other: "", }, VersionStartExcluding: "", VersionStartIncluding: "", VersionEndExcluding: "0.11.0", VersionEndIncluding: "", }, }, constraint: "< 0.11.0", }, { cpe: models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Part: "a", Vendor: "cisco", Product: "node-jose", }, VersionEndIncluding: "0.11.0", }, }, constraint: "<= 0.11.0", }, { cpe: models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Part: "a", Vendor: "cisco", Product: "node-jose", }, VersionStartExcluding: "0.10.0", VersionEndIncluding: "0.11.0", }, }, constraint: "> 0.10.0, <= 0.11.0", }, { cpe: models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Part: "a", Vendor: "cisco", Product: "node-jose", }, VersionStartIncluding: "0.10.0", VersionEndExcluding: "0.11.0", }, }, constraint: ">= 0.10.0, < 0.11.0", }, { cpe: models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Part: "a", Vendor: "cisco", Product: "node-jose", }, VersionStartIncluding: "", VersionEndExcluding: "", }, }, constraint: "", }, } for i, tt := range testdata { constraint := makeVersionConstraint(tt.cpe) if tt.constraint != constraint { t.Errorf("[%d] expected %s, actual %s", i, tt.constraint, constraint) } } } // move to rdb_test.go func TestMatch(t *testing.T) { var testdata = []struct { uri string cpe models.Cpe match bool }{ //0 { uri: "cpe:/a:oracle:vm_virtualbox:5.1.1", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", Version: "", Update: "", Edition: "", Language: "", SoftwareEdition: "", TargetSW: "", TargetHW: "", Other: "", }, VersionStartIncluding: "5.1.0", VersionEndExcluding: "5.1.32", }, }, match: true, }, //1 { uri: "cpe:/a:oracle:vm_virtualbox:5.0.9", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", }, VersionStartIncluding: "5.1.0", VersionEndExcluding: "5.1.32", }, }, match: false, }, //2 { uri: "cpe:/a:oracle:vm_virtualbox:5.1.0", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", }, VersionStartIncluding: "5.1.0", VersionEndExcluding: "5.1.32", }, }, match: true, }, //3 { uri: "cpe:/a:oracle:vm_virtualbox:5.2.32", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", }, VersionStartIncluding: "5.1.0", VersionEndExcluding: "5.1.32", }, }, match: false, }, //4 { uri: "cpe:/a:oracle:vm_virtualbox:5.1.31", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", }, VersionStartIncluding: "5.1.0", VersionEndExcluding: "5.1.32", }, }, match: true, }, //5 { uri: "cpe:/a:oracle:vm_virtualbox:5.1.31", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/a:oracle:vm_virtualbox:5.1.31", CpeWFN: models.CpeWFN{ Part: "a", Vendor: "oracle", Product: "vm_virtualbox", Version: "5.1.31", }, }, }, match: true, }, //6 superset { uri: "cpe:/o:microsoft:windows_7", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/o:microsoft:windows_7::sp1", CpeWFN: models.CpeWFN{ Part: "o", Vendor: "microsoft", Product: "windows_7", Version: "", Update: "sp1", }, }, }, match: true, }, //7 subset { uri: "cpe:/o:microsoft:windows_7::sp2", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/o:microsoft:windows_7", CpeWFN: models.CpeWFN{ Part: "o", Vendor: "microsoft", Product: "windows_7", }, }, }, match: true, }, { uri: "cpe:/o:microsoft:windows_10", cpe: models.Cpe{ CpeBase: models.CpeBase{ URI: "cpe:/o:microsoft:windows_10:-", CpeWFN: models.CpeWFN{ Part: "o", Vendor: "microsoft", Product: "windows_10", Version: "-", }, }, }, match: true, }, } for i, tt := range testdata { uri, err := naming.UnbindURI(tt.uri) match, err := match(uri, tt.cpe) if err != nil { t.Errorf("[%d] err: %s", i, err) } if tt.match != match { t.Errorf("[%d] expected: %t, actual: %t", i, tt.match, match) } } } go-cve-dictionary-0.3.1/db/rdb.go000066400000000000000000000663721337564334100165500ustar00rootroot00000000000000package db import ( "fmt" "io/ioutil" "os" "github.com/cheggaaa/pb" "github.com/jinzhu/gorm" "github.com/knqyf263/go-cpe/common" "github.com/knqyf263/go-cpe/naming" c "github.com/kotakanbe/go-cve-dictionary/config" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" sqlite3 "github.com/mattn/go-sqlite3" // Required MySQL. See http://jinzhu.me/gorm/database.html#connecting-to-a-database _ "github.com/jinzhu/gorm/dialects/mysql" _ "github.com/jinzhu/gorm/dialects/postgres" // Required SQLite3. _ "github.com/jinzhu/gorm/dialects/sqlite" ) // Supported DB dialects. const ( dialectSqlite3 = "sqlite3" dialectMysql = "mysql" dialectPostgreSQL = "postgres" ) // RDBDriver is Driver for RDB type RDBDriver struct { name string conn *gorm.DB } // Name return db name func (r *RDBDriver) Name() string { return r.name } // NewRDB return RDB driver func NewRDB(dbType, dbpath string, debugSQL bool) (driver *RDBDriver, locked bool, err error) { driver = &RDBDriver{ name: dbType, } log.Debugf("Opening DB (%s).", driver.Name()) if locked, err = driver.OpenDB(dbType, dbpath, debugSQL); err != nil { return nil, locked, err } log.Debugf("Migrating DB (%s).", driver.Name()) if err := driver.MigrateDB(); err != nil { return nil, false, err } return driver, false, nil } // OpenDB opens Database func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool) (locked bool, err error) { r.conn, err = gorm.Open(dbType, dbPath) if err != nil { if r.name == dialectSqlite3 { switch err.(sqlite3.Error).Code { case sqlite3.ErrLocked, sqlite3.ErrBusy: return true, err } } return false, fmt.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %s", dbType, dbPath, err) } if err := r.conn.LogMode(debugSQL).Error; err != nil { return false, err } r.conn.LogMode(debugSQL) return false, nil } // MigrateDB migrates Database func (r *RDBDriver) MigrateDB() error { if err := r.conn.AutoMigrate( &models.FeedMeta{}, &models.CveDetail{}, &models.NvdXML{}, &models.NvdJSON{}, &models.Jvn{}, &models.Reference{}, &models.Cpe{}, &models.EnvCpe{}, &models.Cwe{}, &models.Affect{}, &models.Cvss3{}, &models.Cvss2{}, &models.Cvss2Extra{}, &models.Description{}, ).Error; err != nil { return fmt.Errorf("Failed to migrate. err: %s", err) } errs := []error{} // CveID errs = append(errs, r.conn.Model(&models.CveDetail{}). AddIndex("idx_cve_detail_cveid", "cve_id").Error) errs = append(errs, r.conn.Model(&models.NvdXML{}). AddIndex("idx_nvd_xmls_cveid", "cve_id").Error) errs = append(errs, r.conn.Model(&models.NvdJSON{}). AddIndex("idx_nvd_jsons_cveid", "cve_id").Error) errs = append(errs, r.conn.Model(&models.Jvn{}). AddIndex("idx_jvns_cveid", "cve_id").Error) // CveDetailID errs = append(errs, r.conn.Model(&models.NvdXML{}). AddIndex("idx_nvd_xmls_cve_detail_id", "cve_detail_id").Error) errs = append(errs, r.conn.Model(&models.NvdJSON{}). AddIndex("idx_nvd_jsons_cve_detail_id", "cve_detail_id").Error) errs = append(errs, r.conn.Model(&models.Jvn{}). AddIndex("idx_jvns_cve_detail_id", "cve_detail_id").Error) // references errs = append(errs, r.conn.Model(&models.Reference{}). AddIndex("idx_references_nvd_xml_id", "nvd_xml_id").Error) errs = append(errs, r.conn.Model(&models.Reference{}). AddIndex("idx_references_jvn_id", "jvn_id").Error) errs = append(errs, r.conn.Model(&models.Reference{}). AddIndex("idx_references_nvd_json_id", "nvd_json_id").Error) // cpes errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_jvn_id", "jvn_id").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_nvd_xml_id", "nvd_xml_id").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_nvd_json_id", "nvd_json_id").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_uri", "uri").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_formatted_string", "formatted_string").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_part", "part").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_vendor", "vendor").Error) errs = append(errs, r.conn.Model(&models.Cpe{}). AddIndex("idx_cpes_product", "product").Error) // envcpes errs = append(errs, r.conn.Model(&models.EnvCpe{}). AddIndex("idx_envcpes_cpe_id", "cpe_id").Error) errs = append(errs, r.conn.Model(&models.EnvCpe{}). AddIndex("idx_envcpes_uri", "uri").Error) errs = append(errs, r.conn.Model(&models.EnvCpe{}). AddIndex("idx_envcpes_formatted_string", "formatted_string").Error) // Cwes errs = append(errs, r.conn.Model(&models.Cwe{}). AddIndex("idx_cwes_nvd_xml_id", "nvd_xml_id").Error) errs = append(errs, r.conn.Model(&models.Cwe{}). AddIndex("idx_cwes_jvn_id", "jvn_id").Error) errs = append(errs, r.conn.Model(&models.Cwe{}). AddIndex("idx_cwes_nvd_json_id", "nvd_json_id").Error) // Affects //TODO add index to vendor, product if needed errs = append(errs, r.conn.Model(&models.Affect{}). AddIndex("idx_affects_nvd_json_id", "nvd_json_id").Error) // Cvss3 errs = append(errs, r.conn.Model(&models.Cvss3{}). AddIndex("idx_cvss3_nvd_json_id", "nvd_json_id").Error) errs = append(errs, r.conn.Model(&models.Cvss3{}). AddIndex("idx_cvss3_jvn_id", "jvn_id").Error) // Cvss2 errs = append(errs, r.conn.Model(&models.Cvss2{}). AddIndex("idx_cvsss2_nvd_xml_id", "nvd_xml_id").Error) errs = append(errs, r.conn.Model(&models.Cvss2{}). AddIndex("idx_cvss2_jvn_id", "jvn_id").Error) // Cvss2Extra errs = append(errs, r.conn.Model(&models.Cvss2Extra{}). AddIndex("idx_cvsss2_extra_nvd_json_id", "nvd_json_id").Error) // Description errs = append(errs, r.conn.Model(&models.Description{}). AddIndex("idx_descriptions_nvd_json_id", "nvd_json_id").Error) for _, e := range errs { if e != nil { return fmt.Errorf("Failed to create index. err: %s", e) } } return nil } // Get Select Cve information from DB. func (r *RDBDriver) Get(cveID string) (*models.CveDetail, error) { c := models.CveDetail{} err := r.conn.Where(&models.CveDetail{CveID: cveID}).First(&c).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } if c.ID == 0 { return &models.CveDetail{}, nil } // JVN jvn := models.Jvn{} err = r.conn.Model(&c).Related(&jvn, "Jvn").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } if jvn.ID == 0 { c.Jvn = nil } else { c.Jvn = &jvn } if jvn.CveDetailID != 0 && jvn.ID != 0 { jvnRefs := []models.Reference{} err = r.conn.Model(&jvn).Related(&jvnRefs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.Jvn.References = jvnRefs cvss2 := models.Cvss2{} err = r.conn.Model(&jvn).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.Jvn.Cvss2 = cvss2 cvss3 := models.Cvss3{} err = r.conn.Model(&jvn).Related(&cvss3, "Cvss3").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.Jvn.Cvss3 = cvss3 // TODO commentout because JSON response size will be big. so Uncomment if needed. // jvnCpes := []models.Cpe{} // conn.Model(&jvn).Related(&jvnCpes, "Cpes") // c.Jvn.Cpes = jvnCpes } // NVD JSON json := models.NvdJSON{} err = r.conn.Model(&c).Related(&json, "NvdJSON").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } if json.ID == 0 { c.NvdJSON = nil } else { c.NvdJSON = &json } if json.CveDetailID != 0 && json.ID != 0 { descs := []models.Description{} err = r.conn.Model(&json).Related(&descs, "Descriptions").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.Descriptions = descs cvss2 := models.Cvss2Extra{} err = r.conn.Model(&json).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.Cvss2 = cvss2 cvss3 := models.Cvss3{} err = r.conn.Model(&json).Related(&cvss3, "Cvss3").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.Cvss3 = cvss3 cwes := []models.Cwe{} err = r.conn.Model(&json).Related(&cwes, "Cwes").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.Cwes = cwes affects := []models.Affect{} err = r.conn.Model(&json).Related(&affects, "Affects").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.Affects = affects refs := []models.Reference{} err = r.conn.Model(&json).Related(&refs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdJSON.References = refs cpes := []models.Cpe{} err = r.conn.Model(&json).Related(&cpes, "Cpes").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } for i, cpe := range cpes { envCpes := []models.EnvCpe{} err = r.conn.Model(&cpe).Related(&envCpes, "EnvCpes").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } cpes[i].EnvCpes = envCpes } c.NvdJSON.Cpes = cpes } if json.ID != 0 { c.NvdXML = nil return &c, nil } // NVD nvd := models.NvdXML{} err = r.conn.Model(&c).Related(&nvd, "NvdXML").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } if nvd.ID == 0 { c.NvdXML = nil } else { c.NvdXML = &nvd } if nvd.CveDetailID != 0 && nvd.ID != 0 { nvdRefs := []models.Reference{} err = r.conn.Model(&nvd).Related(&nvdRefs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdXML.References = nvdRefs cvss2 := models.Cvss2{} err = r.conn.Model(&nvd).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdXML.Cvss2 = cvss2 cwes := []models.Cwe{} err = r.conn.Model(&nvd).Related(&cwes, "Cwes").Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } c.NvdXML.Cwes = cwes // TODO commentout because JSON response size will be big. so Uncomment if needed. // nvdCpes := []models.Cpe{} // conn.Model(&nvd).Related(&nvdCpes, "Cpes") // c.Nvd.Cpes = nvdCpes } return &c, nil } // GetMulti Select Cves information from DB. func (r *RDBDriver) GetMulti(cveIDs []string) (map[string]models.CveDetail, error) { cveDetails := map[string]models.CveDetail{} for _, cveID := range cveIDs { cve, err := r.Get(cveID) if err != nil { return nil, err } cveDetails[cveID] = *cve } return cveDetails, nil } // CloseDB close Database func (r *RDBDriver) CloseDB() (err error) { if err = r.conn.Close(); err != nil { log.Errorf("Failed to close DB. Type: %s. err: %s", r.name, err) return } return } // GetByCpeURI Select Cve information from DB. func (r *RDBDriver) GetByCpeURI(uri string) ([]models.CveDetail, error) { // parse wfn, get vendor, product specified, err := naming.UnbindURI(uri) if err != nil { return nil, err } // sleect from cpe by vendor, product cpes := []models.Cpe{} err = r.conn.Where(&models.Cpe{ CpeBase: models.CpeBase{ CpeWFN: models.CpeWFN{ Vendor: fmt.Sprintf("%s", specified.Get(common.AttributeVendor)), Product: fmt.Sprintf("%s", specified.Get(common.AttributeProduct)), }, }}).Find(&cpes).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } log.Debugf("specified: %s", uri) filtered := []models.Cpe{} for _, cpe := range cpes { match, err := match(specified, cpe) if err != nil { log.Warnf("Failed to compare the version:%s %s %#v", err, uri, cpe) // continue matching continue } if match { filtered = append(filtered, cpe) } } idDetail := map[uint]models.CveDetail{} for _, cpe := range filtered { var cveDetailID uint if cpe.JvnID != 0 { jvn := models.Jvn{} err = r.conn.Select("cve_detail_id").Where("ID = ?", cpe.JvnID).First(&jvn).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } cveDetailID = jvn.CveDetailID } else if cpe.NvdXMLID != 0 { nvd := models.NvdXML{} err = r.conn.Select("cve_detail_id").Where("ID = ?", cpe.NvdXMLID).First(&nvd).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } cveDetailID = nvd.CveDetailID } else if cpe.NvdJSONID != 0 { json := models.NvdJSON{} err = r.conn.Select("cve_detail_id").Where("ID = ?", cpe.NvdJSONID).First(&json).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } cveDetailID = json.CveDetailID } if _, ok := idDetail[cveDetailID]; ok { continue } cveDetail := models.CveDetail{} err = r.conn.Select("cve_id").Where("ID = ?", cveDetailID).First(&cveDetail).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } cveID := cveDetail.CveID detail, err := r.Get(cveID) if err != nil { return nil, err } idDetail[cveDetailID] = *detail } details := []models.CveDetail{} for _, d := range idDetail { details = append(details, d) log.Debugf("%s", d.CveID) } return details, nil } // InsertJvn inserts Cve Information into DB func (r *RDBDriver) InsertJvn(cves []models.CveDetail) error { log.Infof("Inserting fetched CVEs...") bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() var refreshedJvns []string for chunked := range chunkSlice(cves, 10) { tx := r.conn.Begin() if tx.Error != nil { return tx.Error } for _, c := range chunked { bar.Increment() // select old record. old := models.CveDetail{} result := tx.Where(&models.CveDetail{CveID: c.CveID}).First(&old) if result.Error != nil && result.Error != gorm.ErrRecordNotFound { return rollback(tx, result.Error) } if result.RecordNotFound() || old.ID == 0 { if err := tx.Create(&c).Error; err != nil { return rollback(tx, err) } refreshedJvns = append(refreshedJvns, c.CveID) continue } if !result.RecordNotFound() { // select Jvn from db jvn := models.Jvn{} err := tx.Model(&old).Related(&jvn, "Jvn").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } if jvn.CveDetailID == 0 { c.Jvn.CveDetailID = old.ID if err := tx.Create(&c.Jvn).Error; err != nil { return rollback(tx, err) } refreshedJvns = append(refreshedJvns, c.CveID) continue } // Refresh JVN Record. // skip if the record has already been in DB and not modified. if jvn.LastModifiedDate.Equal(c.Jvn.LastModifiedDate) || jvn.LastModifiedDate.After(c.Jvn.LastModifiedDate) { // log.Debugf("Not modified. old: %s", old.CveID) continue } else { log.Debugf("Newer record found. CveID: %s, old: %s, new: %s", c.CveID, jvn.LastModifiedDate, c.Jvn.LastModifiedDate) } // Delte old References refs := []models.Reference{} err = tx.Model(&jvn).Related(&refs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, r := range refs { if err := tx.Unscoped().Delete(r).Error; err != nil { return rollback(tx, err) } } // Delete Cvss2 cvss2 := []models.Cvss2{} err = tx.Model(&jvn).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cvss := range cvss2 { if err := tx.Unscoped().Delete(cvss).Error; err != nil { return rollback(tx, err) } } // Delete Cvss3 cvss3 := []models.Cvss3{} err = tx.Model(&jvn).Related(&cvss3, "Cvss3").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cvss := range cvss3 { if err := tx.Unscoped().Delete(cvss).Error; err != nil { return rollback(tx, err) } } // Delete old Cpes cpes := []models.Cpe{} err = tx.Model(&jvn).Related(&cpes, "Cpes").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cpe := range cpes { if err := tx.Unscoped().Delete(cpe).Error; err != nil { return rollback(tx, err) } } // Delete old Jvn if err := tx.Unscoped().Delete(&jvn).Error; err != nil { return rollback(tx, err) } // Insert Jvn c.Jvn.CveDetailID = old.ID if err := tx.Create(&c.Jvn).Error; err != nil { return rollback(tx, err) } refreshedJvns = append(refreshedJvns, c.CveID) } } if err := tx.Commit().Error; err != nil { return rollback(tx, err) } } bar.Finish() log.Infof("Refreshed %d Jvns.", len(refreshedJvns)) log.Debugf("%v", refreshedJvns) return nil } func rollback(tx *gorm.DB, err error) error { if err := tx.Rollback().Error; err != nil { return err } return err } // CountNvd count nvd table func (r *RDBDriver) CountNvd() (int, error) { var count int err := r.conn.Model(&models.NvdXML{}).Count(&count).Error if err != nil && err != gorm.ErrRecordNotFound { return 0, err } else if count > 0 { return count, nil } err = r.conn.Model(&models.NvdJSON{}).Count(&count).Error if err != nil && err != gorm.ErrRecordNotFound { return 0, err } return count, nil } // InsertNvdXML inserts CveInformation into DB func (r *RDBDriver) InsertNvdXML(cves []models.CveDetail) error { log.Infof("Inserting CVEs...") bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() var refreshedNvds []string for chunked := range chunkSlice(cves, 10) { tx := r.conn.Begin() if tx.Error != nil { return tx.Error } for _, c := range chunked { bar.Increment() // select old record. old := models.CveDetail{} result := tx.Where(&models.CveDetail{CveID: c.CveID}).First(&old) if result.Error != nil && result.Error != gorm.ErrRecordNotFound { return rollback(tx, result.Error) } if result.RecordNotFound() || old.ID == 0 { if err := tx.Create(&c).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) continue } if !result.RecordNotFound() { // select Nvd from db nvd := models.NvdXML{} err := tx.Model(&old).Related(&nvd, "Nvd").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } if nvd.CveDetailID == 0 { c.NvdXML.CveDetailID = old.ID if err := tx.Create(&c.NvdXML).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) continue } // Refresh to new NVD Record. // skip if the record has already been in DB and not modified. if nvd.LastModifiedDate.Equal(c.NvdXML.LastModifiedDate) || nvd.LastModifiedDate.After(c.NvdXML.LastModifiedDate) { // log.Debugf("Not modified. old: %s", old.CveID) continue } else { log.Debugf("newer Record found. CveID: %s, old: %s, new: %s", c.CveID, nvd.LastModifiedDate, c.NvdXML.LastModifiedDate) } // Delte old References refs := []models.Reference{} err = tx.Model(&nvd).Related(&refs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, r := range refs { if err := tx.Unscoped().Delete(r).Error; err != nil { return rollback(tx, err) } } // Delte old CWE cwes := []models.Cwe{} err = tx.Model(&nvd).Related(&cwes, "Cwes").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cwe := range cwes { if err := tx.Unscoped().Delete(cwe).Error; err != nil { return rollback(tx, err) } } // uncomment if you needed // Delete old Cpes // cpes := []models.Cpe{} // err = tx.Model(&nvd).Related(&cpes, "Cpes").Error // if err != nil && err != gorm.ErrRecordNotFound { // return rollback(tx, err) // } // for _, cpe := range cpes { // if err := tx.Unscoped().Delete(cpe).Error; err != nil { // return rollback(tx, err) // } // } // Delete Cvss2 cvss2 := []models.Cvss2{} err = tx.Model(&nvd).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cvss := range cvss2 { if err := tx.Unscoped().Delete(cvss).Error; err != nil { return rollback(tx, err) } } // Delete old Nvd if err := tx.Unscoped().Delete(&nvd).Error; err != nil { return rollback(tx, err) } // Insert Nvd c.NvdXML.CveDetailID = old.ID if err := tx.Create(&c.NvdXML).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) } } if err := tx.Commit().Error; err != nil { return err } } bar.Finish() log.Infof("Refreshed %d Nvds.", len(refreshedNvds)) // log.Debugf("%v", refreshedNvds) return nil } // InsertNvdJSON Cve information from DB. func (r *RDBDriver) InsertNvdJSON(cves []models.CveDetail) (err error) { log.Infof("Inserting fetched CVEs...") bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() var refreshedNvds []string for chunked := range chunkSlice(cves, 10) { tx := r.conn.Begin() if tx.Error != nil { return tx.Error } for _, c := range chunked { bar.Increment() // select old record. old := models.CveDetail{} result := tx.Where(&models.CveDetail{CveID: c.CveID}).First(&old) if result.Error != nil && result.Error != gorm.ErrRecordNotFound { return rollback(tx, result.Error) } if result.RecordNotFound() || old.ID == 0 { log.Debugf("CveDetail is not found: %s", c.CveID) if err := tx.Create(&c).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) continue } if !result.RecordNotFound() { // select NvdJSON from db json := models.NvdJSON{} err = tx.Model(&old).Related(&json, "NvdJSON").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } // If the NVD JSON hasn't been inserted yet, insert the NVD JSON newly and assosiate it with existing CveDetail. if json.CveDetailID == 0 { log.Debugf("CveDetail found but no NVD JSON: %s", c.CveID) c.NvdJSON.CveDetailID = old.ID if err := tx.Create(&c.NvdJSON).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) continue } // If the NVD JSON has already been inserted, Refresh to new NVD JSON Record. // skip if the record has already been in DB and not modified. if json.LastModifiedDate.Equal(c.NvdJSON.LastModifiedDate) || json.LastModifiedDate.After(c.NvdJSON.LastModifiedDate) { log.Debugf("Not modified. old: %s", old.CveID) continue } else { log.Debugf("newer Record found. CveID: %s, old: %s, new: %s", c.CveID, json.LastModifiedDate, c.NvdJSON.LastModifiedDate) } // Delte old Descriptions descs := []models.Description{} err = tx.Model(&json).Related(&descs, "Descriptions").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, desc := range descs { if err := tx.Unscoped().Delete(desc).Error; err != nil { return rollback(tx, err) } } // Delete Cvss2 cvss2 := []models.Cvss2Extra{} err = tx.Model(&json).Related(&cvss2, "Cvss2").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cvss := range cvss2 { if err := tx.Unscoped().Delete(cvss).Error; err != nil { return rollback(tx, err) } } // Delete Cvss3 cvss3 := []models.Cvss3{} err = tx.Model(&json).Related(&cvss3, "Cvss3").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cvss := range cvss3 { if err := tx.Unscoped().Delete(cvss).Error; err != nil { return rollback(tx, err) } } // Delte old CWE cwes := []models.Cwe{} err = tx.Model(&json).Related(&cwes, "Cwes").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cwe := range cwes { if err := tx.Unscoped().Delete(cwe).Error; err != nil { return rollback(tx, err) } } // Delete old Cpes and EnvEpes cpes := []models.Cpe{} err = tx.Model(&json).Related(&cpes, "Cpes").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, cpe := range cpes { envs := []models.EnvCpe{} err = tx.Model(&cpe).Related(&envs, "EnvCpes").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, env := range envs { if err := tx.Unscoped().Delete(env).Error; err != nil { return rollback(tx, err) } } if err := tx.Unscoped().Delete(cpe).Error; err != nil { return rollback(tx, err) } } // Delte old Affects affects := []models.Affect{} err = tx.Model(&json).Related(&affects, "Affects").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, r := range affects { if err := tx.Unscoped().Delete(r).Error; err != nil { return rollback(tx, err) } } // Delte old References refs := []models.Reference{} err = tx.Model(&json).Related(&refs, "References").Error if err != nil && err != gorm.ErrRecordNotFound { return rollback(tx, err) } for _, r := range refs { if err := tx.Unscoped().Delete(r).Error; err != nil { return rollback(tx, err) } } // Delete old NvdJSON if err := tx.Unscoped().Delete(&json).Error; err != nil { return rollback(tx, err) } // Insert Nvd JSON c.NvdJSON.CveDetailID = old.ID if err := tx.Create(&c.NvdJSON).Error; err != nil { return rollback(tx, err) } refreshedNvds = append(refreshedNvds, c.CveID) } } if err := tx.Commit().Error; err != nil { return rollback(tx, err) } } bar.Finish() log.Infof("Refreshed %d NvdJSONs.", len(refreshedNvds)) // log.Debugf("%v", refreshedNvds) return nil } // GetFetchedFeedMeta selects fetchmeta of the year func (r *RDBDriver) GetFetchedFeedMeta(url string) (*models.FeedMeta, error) { meta := models.FeedMeta{} m := &models.FeedMeta{ URL: url, } err := r.conn.Where(m).First(&meta).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } return &meta, nil } // UpsertFeedHash selects sha1 in metafile of the year func (r *RDBDriver) UpsertFeedHash(mm models.FeedMeta) error { meta := models.FeedMeta{} m := &models.FeedMeta{ URL: mm.URL, } err := r.conn.Where(m).First(&meta).Error if err != nil && err != gorm.ErrRecordNotFound { return err } tx := r.conn.Begin() if tx.Error != nil { return tx.Error } if err == gorm.ErrRecordNotFound { m.Hash = mm.Hash m.LastModifiedDate = mm.LastModifiedDate if err := tx.Create(m).Error; err != nil { return rollback(tx, err) } } else { meta.Hash = mm.Hash m.LastModifiedDate = mm.LastModifiedDate if err := tx.Save(&meta).Error; err != nil { return rollback(tx, err) } } if err := tx.Commit().Error; err != nil { return rollback(tx, err) } return nil } // GetFetchedFeedMetas selects a list of FeedMeta func (r *RDBDriver) GetFetchedFeedMetas() ([]models.FeedMeta, error) { metas := []models.FeedMeta{} err := r.conn.Find(&metas).Error if err != nil && err != gorm.ErrRecordNotFound { return nil, err } return metas, nil } go-cve-dictionary-0.3.1/db/redis.go000066400000000000000000000354511337564334100171010ustar00rootroot00000000000000package db import ( "encoding/json" "fmt" "io/ioutil" "os" "github.com/cheggaaa/pb" "github.com/go-redis/redis" "github.com/knqyf263/go-cpe/common" "github.com/knqyf263/go-cpe/naming" c "github.com/kotakanbe/go-cve-dictionary/config" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" ) /** # Redis Data Structure - HASH ┌──────────────┬──────────┬──────────┬──────────────────────────────────┐ │ HASH │ FIELD │ VALUE │ PURPOSE │ └──────────────┴──────────┴──────────┴──────────────────────────────────┘ ┌──────────────┬──────────┬───────────┬─────────────────────────────────┐ │CVE#${CVEID} │NVD or JVN│${CVEJSON} │Get CVEJSON by CVEID │ ├──────────────┼──────────┼───────────┼─────────────────────────────────┤ │CVE#C#${CVEID}│NVD or JVN│${CPEJSON} │Get CPEJSON BY CVEID │ ├──────────────┼──────────┼───────────┼─────────────────────────────────┤ │ CVE#Meta │${URL} │${METAJSON}│Get FeedMeta BY URL │ └──────────────┴──────────┴───────────┴─────────────────────────────────┘ - ZINDE X ┌─────────────────────────┬──────────┬─────────────┬─────────────────────────────────────┐ │ KEY │ SCORE │ MEMBER │ PURPOSE │ └─────────────────────────┴──────────┴─────────────┴─────────────────────────────────────┘ ┌─────────────────────────┬──────────┬─────────────┬─────────────────────────────────────┐ │CVE#${Vendor}::${Product}│ 0 │[]${CVEID} │Get related []CVEID by Vendor,Product│ └─────────────────────────┴──────────┴─────────────┴─────────────────────────────────────┘ **/ const ( dialectRedis = "redis" hashKeyPrefix = "CVE#" cpeHashKeyPrefix = "CVE#C#" ) // RedisDriver is Driver for Redis type RedisDriver struct { name string conn *redis.Client } // Name return db name func (r *RedisDriver) Name() string { return r.name } // NewRedis return Redis driver func NewRedis(dbType, dbpath string, debugSQL bool) (driver *RedisDriver, locked bool, err error) { driver = &RedisDriver{ name: dbType, } log.Debugf("Opening DB (%s).", driver.Name()) if err = driver.OpenDB(dbType, dbpath, debugSQL); err != nil { return } return } // OpenDB opens Database func (r *RedisDriver) OpenDB(dbType, dbPath string, debugSQL bool) (err error) { var option *redis.Options if option, err = redis.ParseURL(dbPath); err != nil { log.Errorf("%s", err) return fmt.Errorf("Failed to Parse Redis URL. dbpath: %s, err: %s", dbPath, err) } r.conn = redis.NewClient(option) if err = r.conn.Ping().Err(); err != nil { return fmt.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %s", dbType, dbPath, err) } return nil } // CloseDB close Database func (r *RedisDriver) CloseDB() (err error) { if err = r.conn.Close(); err != nil { log.Errorf("Failed to close DB. Type: %s. err: %s", r.name, err) return } return } // Get Select Cve information from DB. func (r *RedisDriver) Get(cveID string) (*models.CveDetail, error) { var cveResult, cpeResult *redis.StringStringMapCmd if cveResult = r.conn.HGetAll(hashKeyPrefix + cveID); cveResult.Err() != nil { return nil, cveResult.Err() } if cpeResult = r.conn.HGetAll(cpeHashKeyPrefix + cveID); cpeResult.Err() != nil { return nil, cpeResult.Err() } return r.unmarshal(cveID, cveResult, cpeResult) } // GetMulti Select Cves information from DB. func (r *RedisDriver) GetMulti(cveIDs []string) (map[string]models.CveDetail, error) { cveDetails := map[string]models.CveDetail{} pipe := r.conn.Pipeline() cveRs, cpeRs := map[string]*redis.StringStringMapCmd{}, map[string]*redis.StringStringMapCmd{} for _, cveID := range cveIDs { cveRs[cveID] = pipe.HGetAll(hashKeyPrefix + cveID) cpeRs[cveID] = pipe.HGetAll(cpeHashKeyPrefix + cveID) } if _, err := pipe.Exec(); err != nil { if err != redis.Nil { return nil, fmt.Errorf("Failed to get multi cve json. err : %s", err) } } for cveID, cveResult := range cveRs { cpeResult := cpeRs[cveID] cveDetail, err := r.unmarshal(cveID, cveResult, cpeResult) if err != nil { return nil, err } cveDetails[cveID] = *cveDetail } return cveDetails, nil } func (r *RedisDriver) unmarshal(cveID string, cveResult, cpeResult *redis.StringStringMapCmd) (*models.CveDetail, error) { var err error jvn := &models.Jvn{} if j, ok := cveResult.Val()["Jvn"]; ok { if err = json.Unmarshal([]byte(j), jvn); err != nil { return nil, err } } if jc, ok := cpeResult.Val()["Jvn"]; ok { if err = json.Unmarshal([]byte(jc), &jvn.Cpes); err != nil { return nil, err } } if jvn.CveID == "" { jvn = nil } nvdjson := &models.NvdJSON{} if j, ok := cveResult.Val()["NvdJSON"]; ok { if err = json.Unmarshal([]byte(j), nvdjson); err != nil { return nil, err } } if jc, ok := cpeResult.Val()["NvdJSON"]; ok { if err = json.Unmarshal([]byte(jc), &nvdjson.Cpes); err != nil { return nil, err } } if nvdjson.CveID == "" { nvdjson = nil } nvdxml := &models.NvdXML{} if nvdjson == nil { if j, ok := cveResult.Val()["Nvd"]; ok { if err = json.Unmarshal([]byte(j), nvdxml); err != nil { return nil, err } } if jc, ok := cpeResult.Val()["Nvd"]; ok { if err = json.Unmarshal([]byte(jc), &nvdxml.Cpes); err != nil { return nil, err } } if nvdxml.CveID == "" { nvdxml = nil } } else { nvdxml = nil } return &models.CveDetail{ CveID: cveID, NvdXML: nvdxml, NvdJSON: nvdjson, Jvn: jvn, }, nil } // GetByCpeURI Select Cve information from DB. func (r *RedisDriver) GetByCpeURI(uri string) ([]models.CveDetail, error) { specified, err := naming.UnbindURI(uri) if err != nil { return nil, err } vendor := fmt.Sprintf("%s", specified.Get(common.AttributeVendor)) product := fmt.Sprintf("%s", specified.Get(common.AttributeProduct)) key := fmt.Sprintf("%s%s::%s", hashKeyPrefix, vendor, product) var result *redis.StringSliceCmd if result = r.conn.ZRange(key, 0, -1); result.Err() != nil { return nil, result.Err() } uniqCveIDs := map[string]bool{} for _, v := range result.Val() { uniqCveIDs[v] = true } details := []models.CveDetail{} for cveID := range uniqCveIDs { d, err := r.Get(cveID) if err != nil { return nil, err } match, err := matchCpe(specified, d) if err != nil { log.Warnf("Failed to compare the version:%s %s %#v", err, uri, d) // continue matching continue } if match { details = append(details, *d) } } return details, nil } // InsertJvn insert items fetched from JVN. func (r *RedisDriver) InsertJvn(cves []models.CveDetail) error { log.Infof("Inserting fetched CVEs...") var err error var refreshedJvns []string bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() for chunked := range chunkSlice(cves, 10) { var pipe redis.Pipeliner pipe = r.conn.Pipeline() for _, c := range chunked { bar.Increment() cpes := make([]models.Cpe, len(c.Jvn.Cpes)) copy(cpes, c.Jvn.Cpes) c.Jvn.Cpes = nil var jj []byte if jj, err = json.Marshal(c.Jvn); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } refreshedJvns = append(refreshedJvns, c.CveID) if result := pipe.HSet(hashKeyPrefix+c.CveID, "Jvn", string(jj)); result.Err() != nil { return fmt.Errorf("Failed to HSet CVE. err: %s", result.Err()) } for _, cpe := range cpes { if result := pipe.ZAdd( fmt.Sprintf("%s%s::%s", hashKeyPrefix, cpe.Vendor, cpe.Product), redis.Z{Score: 0, Member: c.CveID}, ); result.Err() != nil { return fmt.Errorf("Failed to ZAdd cpe. err: %s", result.Err()) } } var jc []byte if jc, err = json.Marshal(cpes); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } if result := pipe.HSet(cpeHashKeyPrefix+c.CveID, "Jvn", string(jc)); result.Err() != nil { return fmt.Errorf("Failed to HSet CPE. err: %s", result.Err()) } } if _, err = pipe.Exec(); err != nil { return fmt.Errorf("Failed to exec pipeline. err: %s", err) } } bar.Finish() log.Infof("Refreshed %d Jvns.", len(refreshedJvns)) // log.Debugf("%v", refreshedJvns) return nil } // CountNvd count nvd table func (r *RedisDriver) CountNvd() (int, error) { var result *redis.StringSliceCmd if result = r.conn.Keys(hashKeyPrefix + "CVE*"); result.Err() != nil { return 0, result.Err() } return len(result.Val()), nil } // InsertNvdXML inserts CveInformation into DB func (r *RedisDriver) InsertNvdXML(cves []models.CveDetail) error { log.Infof("Inserting CVEs...") var err error var refreshedNvds []string bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() for chunked := range chunkSlice(cves, 10) { var pipe redis.Pipeliner pipe = r.conn.Pipeline() for _, c := range chunked { bar.Increment() cpes := make([]models.Cpe, len(c.NvdXML.Cpes)) copy(cpes, c.NvdXML.Cpes) c.NvdXML.Cpes = nil var jn []byte if jn, err = json.Marshal(c.NvdXML); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } refreshedNvds = append(refreshedNvds, c.CveID) if result := pipe.HSet(hashKeyPrefix+c.CveID, "Nvd", string(jn)); result.Err() != nil { return fmt.Errorf("Failed to HSet JVN CVE. err: %s", result.Err()) } for _, cpe := range cpes { if result := pipe.ZAdd( fmt.Sprintf("%s%s::%s", hashKeyPrefix, cpe.Vendor, cpe.Product), redis.Z{Score: 0, Member: c.CveID}, ); result.Err() != nil { return fmt.Errorf("Failed to ZAdd cpe. err: %s", result.Err()) } } var jc []byte if jc, err = json.Marshal(cpes); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } if result := pipe.HSet(cpeHashKeyPrefix+c.CveID, "Nvd", string(jc)); result.Err() != nil { return fmt.Errorf("Failed to HSet NVD CPE. err: %s", result.Err()) } } if _, err = pipe.Exec(); err != nil { return fmt.Errorf("Failed to exec pipeline. err: %s", err) } } bar.Finish() log.Infof("Refreshed %d Nvds.", len(refreshedNvds)) // log.Debugf("%v", refreshedNvds) return nil } // InsertNvdJSON Cve information from DB. func (r *RedisDriver) InsertNvdJSON(cves []models.CveDetail) error { log.Infof("Inserting CVEs...") var err error var refreshedNvds []string bar := pb.New(len(cves)) if c.Conf.Quiet { bar.SetWriter(ioutil.Discard) } else { bar.SetWriter(os.Stderr) } bar.Start() for chunked := range chunkSlice(cves, 10) { var pipe redis.Pipeliner pipe = r.conn.Pipeline() for _, c := range chunked { bar.Increment() cpes := make([]models.Cpe, len(c.NvdJSON.Cpes)) copy(cpes, c.NvdJSON.Cpes) c.NvdJSON.Cpes = nil var jn []byte if jn, err = json.Marshal(c.NvdJSON); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } refreshedNvds = append(refreshedNvds, c.CveID) if result := pipe.HSet(hashKeyPrefix+c.CveID, "NvdJSON", string(jn)); result.Err() != nil { return fmt.Errorf("Failed to HSet CVE. err: %s", result.Err()) } for _, cpe := range cpes { if result := pipe.ZAdd( fmt.Sprintf("%s%s::%s", hashKeyPrefix, cpe.Vendor, cpe.Product), redis.Z{Score: 0, Member: c.CveID}, ); result.Err() != nil { return fmt.Errorf("Failed to ZAdd cpe. err: %s", result.Err()) } } var jc []byte if jc, err = json.Marshal(cpes); err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } if result := pipe.HSet(cpeHashKeyPrefix+c.CveID, "NvdJSON", string(jc)); result.Err() != nil { return fmt.Errorf("Failed to HSet NVD CPE. err: %s", result.Err()) } } if _, err = pipe.Exec(); err != nil { return fmt.Errorf("Failed to exec pipeline. err: %s", err) } } bar.Finish() log.Infof("Refreshed %d Nvds.", len(refreshedNvds)) // log.Debugf("%v", refreshedNvds) return nil } // GetFetchedFeedMeta selects hash in metafile of the year func (r *RedisDriver) GetFetchedFeedMeta(url string) (*models.FeedMeta, error) { var result *redis.StringStringMapCmd if result = r.conn.HGetAll(hashKeyPrefix + "Meta"); result.Err() != nil { return nil, result.Err() } meta := &models.FeedMeta{} if s, ok := result.Val()[url]; ok { if err := json.Unmarshal([]byte(s), meta); err != nil { return nil, err } return meta, nil } return meta, nil } // UpsertFeedHash selects hash in metafile of the year func (r *RedisDriver) UpsertFeedHash(m models.FeedMeta) error { jn, err := json.Marshal(m) if err != nil { return fmt.Errorf("Failed to marshal json. err: %s", err) } var pipe redis.Pipeliner pipe = r.conn.Pipeline() if result := pipe.HSet(hashKeyPrefix+"Meta", m.URL, jn); result.Err() != nil { return fmt.Errorf("Failed to HSet META. err: %s", result.Err()) } if _, err := pipe.Exec(); err != nil { return fmt.Errorf("Failed to exec pipeline. err: %s", err) } return nil } // GetFetchedFeedMetas selects a list of FeedMeta func (r *RedisDriver) GetFetchedFeedMetas() (metas []models.FeedMeta, err error) { var result *redis.StringStringMapCmd if result = r.conn.HGetAll(hashKeyPrefix + "Meta"); result.Err() != nil { return nil, result.Err() } for _, s := range result.Val() { m := models.FeedMeta{} if err := json.Unmarshal([]byte(s), &m); err != nil { return nil, err } metas = append(metas, m) } return } go-cve-dictionary-0.3.1/fetcher/000077500000000000000000000000001337564334100164675ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/fetcher.go000066400000000000000000000054731337564334100204470ustar00rootroot00000000000000package fetcher import ( "bytes" "compress/gzip" "fmt" "io/ioutil" "net/http" "net/url" "time" "github.com/htcat/htcat" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/util" ) //FetchRequest has fetch target information type FetchRequest struct { Year int URL string GZIP bool } //FetchResult has url and fetched Bytes type FetchResult struct { Year int URL string Body []byte } //FetchFeedFiles fetches vulnerability feed file concurrently func FetchFeedFiles(reqs []FetchRequest) (results []FetchResult, err error) { reqChan := make(chan FetchRequest, len(reqs)) resChan := make(chan FetchResult, len(reqs)) errChan := make(chan error, len(reqs)) defer close(reqChan) defer close(resChan) defer close(errChan) for _, r := range reqs { log.Infof("Fetching... %s", r.URL) } go func() { for _, r := range reqs { reqChan <- r } }() concurrency := len(reqs) tasks := util.GenWorkers(concurrency) for range reqs { tasks <- func() { select { case req := <-reqChan: body, err := fetchFile(req, 20/len(reqs)) if err != nil { errChan <- err return } resChan <- FetchResult{ Year: req.Year, URL: req.URL, Body: body, } } return } } errs := []error{} timeout := time.After(10 * 60 * time.Second) for range reqs { select { case res := <-resChan: results = append(results, res) log.Infof("Fetched... %s", res.URL) case err := <-errChan: errs = append(errs, err) case <-timeout: return results, fmt.Errorf("Timeout Fetching") } } if 0 < len(errs) { return results, fmt.Errorf("%s", errs) } return results, nil } func fetchFile(req FetchRequest, parallelism int) (body []byte, err error) { var proxyURL *url.URL httpCilent := &http.Client{} if c.Conf.HTTPProxy != "" { if proxyURL, err = url.Parse(c.Conf.HTTPProxy); err != nil { return nil, fmt.Errorf("Failed to parse proxy url: %s", err) } httpCilent = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} } u, err := url.Parse(req.URL) if err != nil { return nil, fmt.Errorf("aborting: could not parse given URL: %v", err) } buf := bytes.Buffer{} htc := htcat.New(httpCilent, u, parallelism) if _, err := htc.WriteTo(&buf); err != nil { return nil, fmt.Errorf("aborting: could not write to output stream: %v", err) } if req.GZIP { reader, err := gzip.NewReader(bytes.NewReader(buf.Bytes())) defer reader.Close() if err != nil { return nil, fmt.Errorf( "Failed to decompress NVD feedfile. url: %s, err: %s", req.URL, err) } bytes, err := ioutil.ReadAll(reader) if err != nil { return nil, fmt.Errorf( "Failed to Read NVD feedfile. url: %s, err: %s", req.URL, err) } return bytes, nil } return buf.Bytes(), nil } go-cve-dictionary-0.3.1/fetcher/jvn/000077500000000000000000000000001337564334100172645ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/jvn/xml/000077500000000000000000000000001337564334100200645ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/jvn/xml/jvn.go000066400000000000000000000246411337564334100212170ustar00rootroot00000000000000package jvn import ( "encoding/json" "encoding/xml" "fmt" "strconv" "strings" "time" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" "github.com/kotakanbe/go-cve-dictionary/fetcher" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" ) // Meta ... https://jvndb.jvn.jp/ja/feed/checksum.txt type Meta struct { URL string `json:"url"` Hash string `json:"sha256"` LastModified string `json:"lastModified"` } type rdf struct { Items []Item `xml:"item"` } // Item ... http://jvndb.jvn.jp/apis/getVulnOverviewList_api.html type Item struct { About string `xml:"about,attr"` Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` Publisher string `xml:"publisher"` Identifier string `xml:"identifier"` References []references `xml:"references"` Cpes []cpe `xml:"cpe"` Cvsses []Cvss `xml:"cvss"` Date string `xml:"date"` Issued string `xml:"issued"` Modified string `xml:"modified"` } type cpe struct { Version string `xml:"version,attr"` // cpe:/a:mysql:mysql Vendor string `xml:"vendor,attr"` Product string `xml:"product,attr"` Value string `xml:",chardata"` } type references struct { ID string `xml:"id,attr"` Source string `xml:"source,attr"` Title string `xml:"title,attr"` URL string `xml:",chardata"` } // Cvss ... CVSS type Cvss struct { Score string `xml:"score,attr"` Severity string `xml:"severity,attr"` Vector string `xml:"vector,attr"` Version string `xml:"version,attr"` } // ListFetchedFeeds list fetched feeds information func ListFetchedFeeds(driver db.DB) (metas []models.FeedMeta, err error) { lastMetas, err := driver.GetFetchedFeedMetas() if err != nil { return nil, fmt.Errorf("Failed to get Meta: %s", err) } if len(lastMetas) == 0 { log.Infof("No feeds found") return nil, nil } //TODO use meta.Year() uniqYears := map[int]bool{} for _, meta := range lastMetas { if strings.HasSuffix(meta.URL, "jvndb.rdf") || strings.HasSuffix(meta.URL, "jvndb_new.rdf") { uniqYears[c.Latest] = true } else if strings.Contains(meta.URL, "jvndb") { yearstr := strings.TrimSuffix(strings.Split(meta.URL, "jvndb_")[1], ".rdf") y, err := strconv.Atoi(yearstr) if err != nil { return nil, fmt.Errorf("Unable conver to int: %s, err: %s", yearstr, err) } uniqYears[y] = true } } years := []int{} for y := range uniqYears { years = append(years, y) } if len(years) == 0 { return metas, nil } metas, err = FetchLatestFeedMeta(driver, years) if err != nil { return nil, err } return } // FetchLatestFeedMeta Fetch CVE meta information from JVN func FetchLatestFeedMeta(driver db.DB, years []int) (metas []models.FeedMeta, err error) { reqs := []fetcher.FetchRequest{ { URL: "https://jvndb.jvn.jp/ja/feed/checksum.txt", }, } results, err := fetcher.FetchFeedFiles(reqs) if err != nil { return nil, fmt.Errorf("Failed to fetch. err: %s", err) } res := results[0] latestMetas := []Meta{} if err = json.Unmarshal(res.Body, &latestMetas); err != nil { return nil, fmt.Errorf( "Failed to unmarshal. url: %s, err: %s", res.URL, err) } urls := []string{} for _, year := range years { if year == -1 { urls = append(urls, fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/jvndb.rdf"), fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/jvndb_new.rdf")) } else { urls = append(urls, fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/years/jvndb_%d.rdf", year)) } } for _, url := range urls { meta, err := driver.GetFetchedFeedMeta(url) if err != nil { return nil, fmt.Errorf("Failed to get hash: %s, err: %s", url, err) } for _, latestMeta := range latestMetas { if latestMeta.URL == url { meta.URL = url meta.LatestHash = latestMeta.Hash meta.LatestLastModifiedDate = latestMeta.LastModified metas = append(metas, *meta) } } } return } // UpdateMeta updates meta table func UpdateMeta(driver db.DB, metas []models.FeedMeta) error { for _, meta := range metas { meta.Hash = meta.LatestHash meta.LastModifiedDate = meta.LatestLastModifiedDate err := driver.UpsertFeedHash(meta) if err != nil { return fmt.Errorf("Failed to updte meta: %s, err: %s", meta.URL, err) } } return nil } // Fetch fetches vulnerability information from JVN and convert it to model func Fetch(metas []models.FeedMeta) ([]Item, error) { reqs := []fetcher.FetchRequest{} for _, meta := range metas { reqs = append(reqs, fetcher.FetchRequest{ URL: meta.URL, }) } results, err := fetcher.FetchFeedFiles(reqs) if err != nil { return nil, fmt.Errorf("Failed to fetch. err: %s", err) } items := []Item{} for _, res := range results { var rdf rdf if err = xml.Unmarshal([]byte(res.Body), &rdf); err != nil { return nil, fmt.Errorf( "Failed to unmarshal. url: %s, err: %s", res.URL, err) } items = append(items, rdf.Items...) } return items, nil } // FetchConvert fetches vulnerability information from JVN and convert it to model func FetchConvert(metas []models.FeedMeta) (cves []models.CveDetail, err error) { items, err := Fetch(metas) if err != nil { return nil, err } return convert(items) } func convert(items []Item) (cves []models.CveDetail, err error) { for _, item := range items { converted, err := convertToModel(item) if err != nil { return nil, fmt.Errorf("Failed to convert to model. JVN: %s, err: %s", item.Identifier, err) } cves = append(cves, converted...) } return } func makeJvnURLs(years []int) (urls []string) { latestFeeds := []string{ "http://jvndb.jvn.jp/ja/rss/jvndb_new.rdf", "http://jvndb.jvn.jp/ja/rss/jvndb.rdf", } if len(years) == 0 { return latestFeeds } urlFormat := "http://jvndb.jvn.jp/ja/rss/years/jvndb_%d.rdf" for _, year := range years { urls = append(urls, fmt.Sprintf(urlFormat, year)) thisYear := time.Now().Year() if year == thisYear { urls = append(urls, latestFeeds...) } } return } // ConvertJvn converts Jvn structure(got from JVN) to model structure. func convertToModel(item Item) (cves []models.CveDetail, err error) { var cvss2, cvss3 Cvss for _, cvss := range item.Cvsses { switch cvss.Version { case "2.0": cvss2 = cvss case "3.0": cvss3 = cvss } } // References refs := []models.Reference{} for _, r := range item.References { ref := models.Reference{ Source: r.Source, Link: r.URL, } refs = append(refs, ref) } // Cpes cpes := []models.Cpe{} for _, c := range item.Cpes { cpeBase, err := fetcher.ParseCpeURI(c.Value) if err != nil { return nil, err } cpes = append(cpes, models.Cpe{ CpeBase: *cpeBase, }) } publish, err := parseJvnTime(item.Issued) if err != nil { return nil, err } modified, err := parseJvnTime(item.Modified) if err != nil { return nil, err } cveIDs := getCveIDs(item) if len(cveIDs) == 0 { log.Debugf("No CveIDs in references. JvnID: %s, Link: %s", item.Identifier, item.Link) // ignore this item return nil, nil } for _, cveID := range cveIDs { v2elems := parseCvss2VectorStr(cvss2.Vector) v3elems := parseCvss3VectorStr(cvss3.Vector) cve := models.CveDetail{ CveID: cveID, Jvn: &models.Jvn{ CveID: cveID, Title: strings.Replace(item.Title, "\r", "", -1), Summary: strings.Replace(item.Description, "\r", "", -1), JvnLink: item.Link, JvnID: item.Identifier, Cvss2: models.Cvss2{ BaseScore: fetcher.StringToFloat(cvss2.Score), Severity: cvss2.Severity, VectorString: cvss2.Vector, AccessVector: v2elems[0], AccessComplexity: v2elems[1], Authentication: v2elems[2], ConfidentialityImpact: v2elems[3], IntegrityImpact: v2elems[4], AvailabilityImpact: v2elems[5], }, Cvss3: models.Cvss3{ BaseScore: fetcher.StringToFloat(cvss3.Score), BaseSeverity: cvss3.Severity, VectorString: cvss3.Vector, AttackVector: v3elems[0], AttackComplexity: v3elems[1], PrivilegesRequired: v3elems[2], UserInteraction: v3elems[3], Scope: v3elems[4], ConfidentialityImpact: v3elems[5], IntegrityImpact: v3elems[6], AvailabilityImpact: v3elems[7], }, References: refs, Cpes: cpes, PublishedDate: publish, LastModifiedDate: modified, }, } cves = append(cves, cve) } return } var cvss2VectorMap = map[string]string{ "AV:L": "LOCAL", "AV:A": "ADJACENT_NETWORK", "AV:N": "NETWORK", "AC:L": "LOW", "AC:M": "MEDIUM", "AC:H": "HIGH", "Au:M": "MULTIPLE", "Au:S": "SINGLE", "Au:N": "NONE", "C:N": "NONE", "C:P": "PARTIAL", "C:C": "COMPLETE", "I:N": "NONE", "I:P": "PARTIAL", "I:C": "COMPLETE", "A:N": "NONE", "A:P": "PARTIAL", "A:C": "COMPLETE", } func parseCvss2VectorStr(str string) (elems []string) { if len(str) == 0 { return []string{"", "", "", "", "", ""} } for _, s := range strings.Split(str, "/") { elems = append(elems, cvss2VectorMap[s]) } return } var cvss3VectorMap = map[string]string{ "AV:N": "NETWORK", "AV:A": "ADJACENT_NETWORK", "AV:L": "LOCAL", "AV:P": "PHYSICAL", "AC:L": "LOW", "AC:H": "HIGH", "PR:N": "NONE", "PR:L": "LOW", "PR:H": "HIGH", "UI:N": "NONE", "UI:R": "REQUIRED", "S:U": "UNCHANGED", "S:C": "CHANGED", "C:N": "NONE", "C:L": "LOW", "C:H": "HIGH", "I:N": "NONE", "I:L": "LOW", "I:H": "HIGH", "A:N": "NONE", "A:L": "LOW", "A:H": "HIGH", } func parseCvss3VectorStr(str string) (elems []string) { if len(str) == 0 { return []string{"", "", "", "", "", "", "", ""} } str = strings.TrimPrefix(str, "CVSS:3.0/") for _, s := range strings.Split(str, "/") { elems = append(elems, cvss3VectorMap[s]) } return } // convert string time to time.Time // JVN : "2016-01-26T13:36:23+09:00", // NVD : "2016-01-20T21:59:01.313-05:00", func parseJvnTime(strtime string) (t time.Time, err error) { layout := "2006-01-02T15:04-07:00" t, err = time.Parse(layout, strtime) if err != nil { return t, fmt.Errorf("Failed to parse time, time: %s, err: %s", strtime, err) } return } func getCveIDs(item Item) []string { cveIDsMap := map[string]bool{} for _, ref := range item.References { switch ref.Source { case "NVD", "CVE": cveIDsMap[ref.ID] = true } } var cveIDs []string for cveID := range cveIDsMap { cveIDs = append(cveIDs, cveID) } return cveIDs } go-cve-dictionary-0.3.1/fetcher/nvd/000077500000000000000000000000001337564334100172565ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/nvd/json/000077500000000000000000000000001337564334100202275ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/nvd/json/nvd.go000066400000000000000000000314701337564334100213520ustar00rootroot00000000000000package json import ( "encoding/json" "fmt" "strings" "time" version "github.com/hashicorp/go-version" "github.com/k0kubun/pp" "github.com/kotakanbe/go-cve-dictionary/fetcher" "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" ) // FetchConvert Fetch CVE vulnerability information from NVD func FetchConvert(metas []models.FeedMeta) (cves []models.CveDetail, err error) { reqs := []fetcher.FetchRequest{} for _, meta := range metas { reqs = append(reqs, fetcher.FetchRequest{ URL: meta.URL, GZIP: true, }) } results, err := fetcher.FetchFeedFiles(reqs) if err != nil { return nil, fmt.Errorf("Failed to fetch. err: %s", err) } for _, res := range results { nvd := NvdJSON{} if err = json.Unmarshal(res.Body, &nvd); err != nil { return nil, fmt.Errorf( "Failed to unmarshal. url: %s, err: %s", res.URL, err) } for _, item := range nvd.CveItems { cve, err := convertToModel(&item) if err != nil { return nil, fmt.Errorf("Failed to convert to model. cve: %s, err: %s", item.Cve.CveDataMeta.ID, err) } cves = append(cves, *cve) } } return } // NvdJSON is a struct of NVD JSON // https://scap.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema type NvdJSON struct { CveDataType string `json:"CVE_data_type"` CveDataFormat string `json:"CVE_data_format"` CveDataVersion string `json:"CVE_data_version"` CveDataNumberOfCVEs string `json:"CVE_data_numberOfCVEs"` CveDataTimestamp string `json:"CVE_data_timestamp"` CveItems []CveItem `json:"CVE_Items"` } // CveItem is a struct of NvdJSON>CveItems type CveItem struct { Cve struct { DataType string `json:"data_type"` DataFormat string `json:"data_format"` DataVersion string `json:"data_version"` CveDataMeta struct { ID string `json:"ID"` ASSIGNER string `json:"ASSIGNER"` } `json:"CVE_data_meta"` Affects struct { Vendor struct { VendorData []struct { VendorName string `json:"vendor_name"` Product struct { ProductData []struct { ProductName string `json:"product_name"` Version struct { VersionData []struct { VersionValue string `json:"version_value"` } `json:"version_data"` } `json:"version"` } `json:"product_data"` } `json:"product"` } `json:"vendor_data"` } `json:"vendor"` } `json:"affects"` Problemtype struct { ProblemtypeData []struct { Description []struct { Lang string `json:"lang"` Value string `json:"value"` } `json:"description"` } `json:"problemtype_data"` } `json:"problemtype"` References struct { ReferenceData []struct { URL string `json:"url"` } `json:"reference_data"` } `json:"references"` Description struct { DescriptionData []struct { Lang string `json:"lang"` Value string `json:"value"` } `json:"description_data"` } `json:"description"` } `json:"cve"` Configurations struct { CveDataVersion string `json:"CVE_data_version"` Nodes []struct { Operator string `json:"operator"` Negate bool `json:"negate"` Cpes []struct { Vulnerable bool `json:"vulnerable"` Cpe23URI string `json:"cpe23Uri"` VersionStartExcluding string `json:"versionStartExcluding"` VersionStartIncluding string `json:"versionStartIncluding"` VersionEndExcluding string `json:"versionEndExcluding"` VersionEndIncluding string `json:"versionEndIncluding"` } `json:"cpe_match"` Children []struct { Operator string `json:"operator"` Cpes []struct { Vulnerable bool `json:"vulnerable"` Cpe23URI string `json:"cpe23Uri"` VersionStartExcluding string `json:"versionStartExcluding"` VersionStartIncluding string `json:"versionStartIncluding"` VersionEndExcluding string `json:"versionEndExcluding"` VersionEndIncluding string `json:"versionEndIncluding"` } `json:"cpe_match"` } `json:"children,omitempty"` } `json:"nodes"` } `json:"configurations"` Impact struct { BaseMetricV3 struct { CvssV3 struct { Version string `json:"version"` VectorString string `json:"vectorString"` AttackVector string `json:"attackVector"` AttackComplexity string `json:"attackComplexity"` PrivilegesRequired string `json:"privilegesRequired"` UserInteraction string `json:"userInteraction"` Scope string `json:"scope"` ConfidentialityImpact string `json:"confidentialityImpact"` IntegrityImpact string `json:"integrityImpact"` AvailabilityImpact string `json:"availabilityImpact"` BaseScore float64 `json:"baseScore"` BaseSeverity string `json:"baseSeverity"` } `json:"cvssV3"` ExploitabilityScore float64 `json:"exploitabilityScore"` ImpactScore float64 `json:"impactScore"` } `json:"baseMetricV3"` BaseMetricV2 struct { CvssV2 struct { Version string `json:"version"` VectorString string `json:"vectorString"` AccessVector string `json:"accessVector"` AccessComplexity string `json:"accessComplexity"` Authentication string `json:"authentication"` ConfidentialityImpact string `json:"confidentialityImpact"` IntegrityImpact string `json:"integrityImpact"` AvailabilityImpact string `json:"availabilityImpact"` BaseScore float64 `json:"baseScore"` } `json:"cvssV2"` Severity string `json:"severity"` ExploitabilityScore float64 `json:"exploitabilityScore"` ImpactScore float64 `json:"impactScore"` ObtainAllPrivilege bool `json:"obtainAllPrivilege"` ObtainUserPrivilege bool `json:"obtainUserPrivilege"` ObtainOtherPrivilege bool `json:"obtainOtherPrivilege"` UserInteractionRequired bool `json:"userInteractionRequired"` } `json:"baseMetricV2"` } `json:"impact"` PublishedDate string `json:"publishedDate"` LastModifiedDate string `json:"lastModifiedDate"` } // convertToModel converts Nvd JSON to model structure. func convertToModel(item *CveItem) (*models.CveDetail, error) { //References refs := []models.Reference{} for _, r := range item.Cve.References.ReferenceData { ref := models.Reference{ Link: r.URL, } refs = append(refs, ref) } // Cwes cwes := []models.Cwe{} for _, data := range item.Cve.Problemtype.ProblemtypeData { for _, desc := range data.Description { cwes = append(cwes, models.Cwe{ CweID: desc.Value, }) } } // Affects affects := []models.Affect{} for _, vendor := range item.Cve.Affects.Vendor.VendorData { for _, prod := range vendor.Product.ProductData { for _, version := range prod.Version.VersionData { affects = append(affects, models.Affect{ Vendor: vendor.VendorName, Product: prod.ProductName, Version: version.VersionValue, }) } } } // Traverse Cpe, EnvCpe cpes := []models.Cpe{} for _, node := range item.Configurations.Nodes { if node.Negate { continue } nodeCpes := []models.Cpe{} for _, cpe := range node.Cpes { if !cpe.Vulnerable { // CVE-2017-14492 and CVE-2017-8581 has a cpe that has vulenrable:false. // But these vulnerable: false cpe is also vulnerable... // So, ignore the vulerable flag of this layer(under nodes>cpe) } cpeBase, err := fetcher.ParseCpeURI(cpe.Cpe23URI) if err != nil { // logging only log.Infof("Failed to parse CpeURI %s: %s", cpe.Cpe23URI, err) continue } cpeBase.VersionStartExcluding = cpe.VersionStartExcluding cpeBase.VersionStartIncluding = cpe.VersionStartIncluding cpeBase.VersionEndExcluding = cpe.VersionEndExcluding cpeBase.VersionEndIncluding = cpe.VersionEndIncluding nodeCpes = append(nodeCpes, models.Cpe{ CpeBase: *cpeBase, }) if !checkIfVersionParsable(cpeBase) { return nil, fmt.Errorf( "Version parse err. Please add a issue on [GitHub](https://github.com/kotakanbe/go-cve-dictionary/issues/new). Title: %s, Content:%s", item.Cve.CveDataMeta.ID, pp.Sprintf("%v", *item), ) } } for _, child := range node.Children { for _, cpe := range child.Cpes { if cpe.Vulnerable { cpeBase, err := fetcher.ParseCpeURI(cpe.Cpe23URI) if err != nil { return nil, err } cpeBase.VersionStartExcluding = cpe.VersionStartExcluding cpeBase.VersionStartIncluding = cpe.VersionStartIncluding cpeBase.VersionEndExcluding = cpe.VersionEndExcluding cpeBase.VersionEndIncluding = cpe.VersionEndIncluding nodeCpes = append(nodeCpes, models.Cpe{ CpeBase: *cpeBase, }) if !checkIfVersionParsable(cpeBase) { return nil, fmt.Errorf( "Version parse err. Please add a issue on [GitHub](https://github.com/kotakanbe/go-cve-dictionary/issues/new). Title: %s, Content:%s", item.Cve.CveDataMeta.ID, pp.Sprintf("%v", *item), ) } } else { if node.Operator == "AND" { for i, c := range nodeCpes { cpeBase, err := fetcher.ParseCpeURI(cpe.Cpe23URI) if err != nil { return nil, err } cpeBase.VersionStartExcluding = cpe.VersionStartExcluding cpeBase.VersionStartIncluding = cpe.VersionStartIncluding cpeBase.VersionEndExcluding = cpe.VersionEndExcluding cpeBase.VersionEndIncluding = cpe.VersionEndIncluding nodeCpes[i].EnvCpes = append(c.EnvCpes, models.EnvCpe{ CpeBase: *cpeBase, }) if !checkIfVersionParsable(cpeBase) { return nil, fmt.Errorf( "Please add a issue on [GitHub](https://github.com/kotakanbe/go-cve-dictionary/issues/new). Title: Version parse err: %s, Content:%s", item.Cve.CveDataMeta.ID, pp.Sprintf("%v", *item), ) } } } } } } cpes = append(cpes, nodeCpes...) } // Description descs := []models.Description{} for _, desc := range item.Cve.Description.DescriptionData { descs = append(descs, models.Description{ Lang: desc.Lang, Value: desc.Value, }) } publish, err := parseNvdJSONTime(item.PublishedDate) if err != nil { return nil, err } modified, err := parseNvdJSONTime(item.LastModifiedDate) if err != nil { return nil, err } c2 := item.Impact.BaseMetricV2 c3 := item.Impact.BaseMetricV3 return &models.CveDetail{ CveID: item.Cve.CveDataMeta.ID, NvdJSON: &models.NvdJSON{ CveID: item.Cve.CveDataMeta.ID, Descriptions: descs, Cvss2: models.Cvss2Extra{ Cvss2: models.Cvss2{ VectorString: c2.CvssV2.VectorString, AccessVector: c2.CvssV2.AccessVector, AccessComplexity: c2.CvssV2.AccessComplexity, Authentication: c2.CvssV2.Authentication, ConfidentialityImpact: c2.CvssV2.ConfidentialityImpact, IntegrityImpact: c2.CvssV2.IntegrityImpact, AvailabilityImpact: c2.CvssV2.AvailabilityImpact, BaseScore: c2.CvssV2.BaseScore, Severity: c2.Severity, }, ExploitabilityScore: c2.ExploitabilityScore, ImpactScore: c2.ImpactScore, ObtainAllPrivilege: c2.ObtainAllPrivilege, ObtainUserPrivilege: c2.ObtainUserPrivilege, ObtainOtherPrivilege: c2.ObtainOtherPrivilege, UserInteractionRequired: c2.UserInteractionRequired, }, Cvss3: models.Cvss3{ VectorString: c3.CvssV3.VectorString, AttackVector: c3.CvssV3.AttackVector, AttackComplexity: c3.CvssV3.AttackComplexity, PrivilegesRequired: c3.CvssV3.PrivilegesRequired, UserInteraction: c3.CvssV3.UserInteraction, Scope: c3.CvssV3.Scope, ConfidentialityImpact: c3.CvssV3.ConfidentialityImpact, IntegrityImpact: c3.CvssV3.IntegrityImpact, AvailabilityImpact: c3.CvssV3.AvailabilityImpact, BaseScore: c3.CvssV3.BaseScore, BaseSeverity: c3.CvssV3.BaseSeverity, ExploitabilityScore: c3.ExploitabilityScore, ImpactScore: c3.ImpactScore, }, Cwes: cwes, Cpes: cpes, References: refs, Affects: affects, PublishedDate: publish, LastModifiedDate: modified, }, }, nil } func checkIfVersionParsable(cpeBase *models.CpeBase) bool { if cpeBase.Version != "ANY" && cpeBase.Version != "NA" { vers := []string{cpeBase.VersionStartExcluding, cpeBase.VersionStartIncluding, cpeBase.VersionEndIncluding, cpeBase.VersionEndExcluding} for _, v := range vers { if v == "" { continue } v := strings.Replace(v, `\`, "", -1) if _, err := version.NewVersion(v); err != nil { return false } } } return true } func parseNvdJSONTime(strtime string) (t time.Time, err error) { layout := "2006-01-02T15:04Z" t, err = time.Parse(layout, strtime) if err != nil { return t, fmt.Errorf("Failed to parse time, time: %s, err: %s", strtime, err) } return } go-cve-dictionary-0.3.1/fetcher/nvd/util.go000066400000000000000000000101731337564334100205640ustar00rootroot00000000000000package nvd import ( "fmt" "strconv" "strings" c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" "github.com/kotakanbe/go-cve-dictionary/fetcher" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" ) // ListFetchedFeeds list fetched feeds information func ListFetchedFeeds(driver db.DB) (jsonMetas, xmlMetas []models.FeedMeta, err error) { lastMetas, err := driver.GetFetchedFeedMetas() if err != nil { return nil, nil, fmt.Errorf("Failed to get Meta: %s", err) } if len(lastMetas) == 0 { log.Infof("No feeds found") return } xmlYears, jsonYears := map[int]bool{}, map[int]bool{} for _, meta := range lastMetas { if !checkNvdURL(meta.URL) { continue } y, xml, err := nvdFeedURLToYear(meta.URL) if err != nil { return nil, nil, err } if xml { xmlYears[y] = true } else { jsonYears[y] = true } } xmlY, jsonY := []int{}, []int{} for y := range xmlYears { xmlY = append(xmlY, y) } for y := range jsonYears { jsonY = append(jsonY, y) } xmlMetas, err = FetchLatestFeedMeta(driver, xmlY, true) if err != nil { return nil, nil, err } jsonMetas, err = FetchLatestFeedMeta(driver, jsonY, false) if err != nil { return nil, nil, err } return } func checkNvdURL(url string) bool { return strings.Contains(url, "nvdcve-") } func nvdFeedURLToYear(url string) (year int, xml bool, err error) { //TODO use meta.Year() yearstr := "" if strings.Contains(url, "nvdcve-2.0-") { xml = true yearstr = strings.TrimSuffix(strings.Split(url, "nvdcve-2.0-")[1], ".xml.gz") } else if strings.Contains(url, "nvdcve-1.0-") { yearstr = strings.TrimSuffix(strings.Split(url, "nvdcve-1.0-")[1], ".json.gz") } else { return year, xml, fmt.Errorf("Failed to parse URL: %s", url) } switch yearstr { case "recent", "modified": return c.Latest, xml, nil default: y, err := strconv.Atoi(yearstr) if err != nil { return 0, false, fmt.Errorf("Unable conver to int: %d, err: %s", year, err) } return y, xml, nil } } // FetchLatestFeedMeta fetches CVE meta information from NVD func FetchLatestFeedMeta(driver db.DB, years []int, xml bool) (metas []models.FeedMeta, err error) { reqs := []fetcher.FetchRequest{} for _, year := range years { urls := MakeNvdMetaURLs(year, xml) for _, url := range urls { reqs = append(reqs, fetcher.FetchRequest{ Year: year, URL: url, }) } } results, err := fetcher.FetchFeedFiles(reqs) if err != nil { return nil, fmt.Errorf("Failed to fetch. err: %s", err) } for _, res := range results { str := string(res.Body) ss := strings.Split(str, "\r\n") if len(ss) != 6 { continue } hash := ss[4] url := "" if xml { url = strings.Replace(res.URL, ".meta", ".xml.gz", -1) } else { url = strings.Replace(res.URL, ".meta", ".json.gz", -1) } meta, err := driver.GetFetchedFeedMeta(url) if err != nil { return nil, fmt.Errorf("Failed to get meta: %d, err: %s", res.Year, err) } meta.URL = url meta.LatestHash = hash meta.LatestLastModifiedDate = strings.TrimPrefix(ss[0], "lastModifiedDate:") metas = append(metas, *meta) } return } // UpdateMeta updates meta table func UpdateMeta(driver db.DB, metas []models.FeedMeta) error { for _, meta := range metas { meta.Hash = meta.LatestHash meta.LastModifiedDate = meta.LatestLastModifiedDate err := driver.UpsertFeedHash(meta) if err != nil { return fmt.Errorf("Failed to updte meta: %s, err: %s", meta.URL, err) } } return nil } // MakeNvdMetaURLs returns a URL of NVD Feed func MakeNvdMetaURLs(year int, xml bool) (url []string) { formatTemplate := "" if xml { // https://nvd.nist.gov/vuln/data-feeds#XML_FEED formatTemplate = "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" } else { // https: //nvd.nist.gov/vuln/data-feeds#JSON_FEED formatTemplate = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-%s.meta" } if year == c.Latest { for _, name := range []string{"modified", "recent"} { url = append(url, fmt.Sprintf(formatTemplate, name)) } } else { feed := strconv.Itoa(year) url = append(url, fmt.Sprintf(formatTemplate, feed)) } return } go-cve-dictionary-0.3.1/fetcher/nvd/util_test.go000066400000000000000000000041651337564334100216270ustar00rootroot00000000000000package nvd import ( "reflect" "testing" c "github.com/kotakanbe/go-cve-dictionary/config" ) func TestNvdFeedURLToYear(t *testing.T) { var tests = []struct { in string year int xml bool }{ { in: "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2018.json.gz", year: 2018, xml: false, }, { in: "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.gz", year: c.Latest, xml: false, }, { in: "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.gz", year: c.Latest, xml: false, }, { in: "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2018.xml.gz", year: 2018, xml: true, }, { in: "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-recent.xml.gz", year: c.Latest, xml: true, }, { in: "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml.gz", year: c.Latest, xml: true, }, } for i, tt := range tests { y, xml, err := nvdFeedURLToYear(tt.in) if err != nil { t.Errorf("[%d] err: %s", i, err) } if y != tt.year { t.Errorf("[%d] expected: %v\n actual: %v\n", i, y, tt.year) } if xml != tt.xml { t.Errorf("[%d] expected: %v\n actual: %v\n", i, y, tt.xml) } } } func TestMakeNvdMetaURL(t *testing.T) { var tests = []struct { year int xml bool url []string }{ { year: 2018, xml: false, url: []string{"https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2018.meta"}, }, { year: c.Latest, xml: false, url: []string{ "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.meta", "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.meta", }, }, { year: 2018, xml: true, url: []string{"https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2018.meta"}, }, { year: c.Latest, xml: true, url: []string{ "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.meta", "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-recent.meta", }, }, } for i, tt := range tests { url := MakeNvdMetaURLs(tt.year, tt.xml) if !reflect.DeepEqual(url, tt.url) { t.Errorf("[%d] expected: %v\n actual: %v\n", i, tt.url, url) } } } go-cve-dictionary-0.3.1/fetcher/nvd/xml/000077500000000000000000000000001337564334100200565ustar00rootroot00000000000000go-cve-dictionary-0.3.1/fetcher/nvd/xml/nvd.go000066400000000000000000000110471337564334100211770ustar00rootroot00000000000000package xml import ( "encoding/xml" "fmt" "time" "github.com/kotakanbe/go-cve-dictionary/fetcher" "github.com/kotakanbe/go-cve-dictionary/models" ) // NvdXML is array of Entry type NvdXML struct { Entries []Entry `xml:"entry"` } // Entry is Root Element type Entry struct { CveID string `xml:"id,attr" json:"id"` PublishedDate time.Time `xml:"published-datetime"` LastModifiedDate time.Time `xml:"last-modified-datetime"` Cvss Cvss `xml:"cvss>base_metrics" json:"cvss"` Products []string `xml:"vulnerable-software-list>product"` //CPE Summary string `xml:"summary"` References []Reference `xml:"references"` Cwes []Cwe `xml:"cwe"` } // Cvss is Cvss Score type Cvss struct { Score string `xml:"score"` AccessVector string `xml:"access-vector"` AccessComplexity string `xml:"access-complexity"` Authentication string `xml:"authentication"` ConfidentialityImpact string `xml:"confidentiality-impact"` IntegrityImpact string `xml:"integrity-impact"` AvailabilityImpact string `xml:"availability-impact"` Source string `xml:"source"` GeneratedOnDate time.Time `xml:"generated-on-datetime"` } // Cwe has Cwe ID type Cwe struct { ID string `xml:"id,attr"` } // Reference is additional information about the CVE type Reference struct { Type string `xml:"reference_type,attr"` Source string `xml:"source"` Link Link `xml:"reference"` } // Link is additional information about the CVE type Link struct { Value string `xml:",chardata" json:"value"` Href string `xml:"href,attr" json:"href"` } // FetchConvert Fetch CVE vulnerability informatino from JVN func FetchConvert(metas []models.FeedMeta) (cves []models.CveDetail, err error) { reqs := []fetcher.FetchRequest{} for _, meta := range metas { reqs = append(reqs, fetcher.FetchRequest{ URL: meta.URL, GZIP: true, }) } results, err := fetcher.FetchFeedFiles(reqs) if err != nil { return nil, fmt.Errorf("Failed to fetch. err: %s", err) } for _, res := range results { nvd := NvdXML{} if err = xml.Unmarshal(res.Body, &nvd); err != nil { return nil, fmt.Errorf( "Failed to unmarshal. url: %s, err: %s", res.URL, err) } for _, e := range nvd.Entries { cve, err := convertToModel(e) if err != nil { return nil, fmt.Errorf("Failed to convert to model. cve: %s, err: %s", e.CveID, err) } cves = append(cves, *cve) } } return } // converToModel converts Nvd structure(got from NVD) to model structure. func convertToModel(entry Entry) (cves *models.CveDetail, err error) { refs := []models.Reference{} for _, r := range entry.References { ref := models.Reference{ Source: r.Source, Link: r.Link.Href, } refs = append(refs, ref) } // uncomment if you needed // cpes := []models.Cpe{} // for _, c := range entry.Products { // cpeBase, err := fetcher.ParseCpeURI(c) // if err != nil { // return nil, err // } // cpes = append(cpes, models.Cpe{ // CpeBase: *cpeBase, // }) // } cwes := []models.Cwe{} for _, cwe := range entry.Cwes { cwes = append(cwes, models.Cwe{ CweID: cwe.ID, }) } score := fetcher.StringToFloat(entry.Cvss.Score) vectorString := "" if entry.Cvss.AccessVector != "" { vectorString = fmt.Sprintf("AV:%c/AC:%c/Au:%c/C:%c/I:%c/A:%c", entry.Cvss.AccessVector[0], entry.Cvss.AccessComplexity[0], entry.Cvss.Authentication[0], entry.Cvss.ConfidentialityImpact[0], entry.Cvss.IntegrityImpact[0], entry.Cvss.AvailabilityImpact[0], ) } return &models.CveDetail{ CveID: entry.CveID, NvdXML: &models.NvdXML{ CveID: entry.CveID, Summary: entry.Summary, Cvss2: models.Cvss2{ BaseScore: score, Severity: cvss2ScoreToSeverity(score), VectorString: vectorString, AccessVector: entry.Cvss.AccessVector, AccessComplexity: entry.Cvss.AccessComplexity, Authentication: entry.Cvss.Authentication, ConfidentialityImpact: entry.Cvss.ConfidentialityImpact, IntegrityImpact: entry.Cvss.IntegrityImpact, AvailabilityImpact: entry.Cvss.AvailabilityImpact, }, Cwes: cwes, PublishedDate: entry.PublishedDate, LastModifiedDate: entry.LastModifiedDate, // Cpes: cpes, References: refs, }, }, nil } // https://nvd.nist.gov/vuln-metrics/cvss func cvss2ScoreToSeverity(score float64) string { if 7.0 <= score { return "HIGH" } else if 4.0 <= score { return "MEDIUM" } else { return "LOW" } } go-cve-dictionary-0.3.1/fetcher/nvd/xml/nvd_test.go000066400000000000000000000000141337564334100222260ustar00rootroot00000000000000package xml go-cve-dictionary-0.3.1/fetcher/util.go000066400000000000000000000040151337564334100177730ustar00rootroot00000000000000package fetcher import ( "fmt" "strconv" "strings" "github.com/knqyf263/go-cpe/common" "github.com/knqyf263/go-cpe/naming" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/kotakanbe/go-cve-dictionary/models" ) // ParseCpeURI parses cpe22uri and set to models.CpeBase func ParseCpeURI(uri string) (*models.CpeBase, error) { var wfn common.WellFormedName var err error if strings.HasPrefix(uri, "cpe:/") { val := strings.TrimPrefix(uri, "cpe:/") if strings.Contains(val, "/") { uri = "cpe:/" + strings.Replace(val, "/", `\/`, -1) } wfn, err = naming.UnbindURI(uri) if err != nil { return nil, err } } else { wfn, err = naming.UnbindFS(uri) if err != nil { return nil, err } } return &models.CpeBase{ URI: naming.BindToURI(wfn), FormattedString: naming.BindToFS(wfn), WellFormedName: wfn.String(), CpeWFN: models.CpeWFN{ Part: fmt.Sprintf("%s", wfn.Get(common.AttributePart)), Vendor: fmt.Sprintf("%s", wfn.Get(common.AttributeVendor)), Product: fmt.Sprintf("%s", wfn.Get(common.AttributeProduct)), Version: fmt.Sprintf("%s", wfn.Get(common.AttributeVersion)), Update: fmt.Sprintf("%s", wfn.Get(common.AttributeUpdate)), Edition: fmt.Sprintf("%s", wfn.Get(common.AttributeEdition)), Language: fmt.Sprintf("%s", wfn.Get(common.AttributeLanguage)), SoftwareEdition: fmt.Sprintf("%s", wfn.Get(common.AttributeSwEdition)), TargetSW: fmt.Sprintf("%s", wfn.Get(common.AttributeTargetSw)), TargetHW: fmt.Sprintf("%s", wfn.Get(common.AttributeTargetHw)), Other: fmt.Sprintf("%s", wfn.Get(common.AttributeOther)), }, }, nil } // StringToFloat cast string to float64 func StringToFloat(str string) float64 { if len(str) == 0 { return 0 } var f float64 var ignorableError error if f, ignorableError = strconv.ParseFloat(str, 64); ignorableError != nil { log.Errorf("Failed to cast CVSS score. score: %s, err; %s", str, ignorableError, ) f = 0 } return f } go-cve-dictionary-0.3.1/log/000077500000000000000000000000001337564334100156305ustar00rootroot00000000000000go-cve-dictionary-0.3.1/log/log.go000066400000000000000000000034251337564334100167440ustar00rootroot00000000000000package log import ( "fmt" "io/ioutil" "os" "path/filepath" logger "github.com/inconshreveable/log15" "github.com/k0kubun/pp" ) // SetLogger set logger func SetLogger(logDir string, quiet, debug, logJSON bool) { stderrHundler := logger.StderrHandler logFormat := logger.LogfmtFormat() if logJSON { logFormat = logger.JsonFormatEx(false, true) stderrHundler = logger.StreamHandler(os.Stderr, logFormat) } lvlHundler := logger.LvlFilterHandler(logger.LvlInfo, stderrHundler) if debug { lvlHundler = logger.LvlFilterHandler(logger.LvlDebug, stderrHundler) } if quiet { lvlHundler = logger.LvlFilterHandler(logger.LvlDebug, logger.DiscardHandler()) pp.SetDefaultOutput(ioutil.Discard) } if _, err := os.Stat(logDir); os.IsNotExist(err) { if err := os.Mkdir(logDir, 0700); err != nil { logger.Error("Failed to create log directory", "err", err) } } var hundler logger.Handler if _, err := os.Stat(logDir); err == nil { logPath := filepath.Join(logDir, "cve-dictionary.log") hundler = logger.MultiHandler( logger.Must.FileHandler(logPath, logFormat), lvlHundler, ) } else { hundler = lvlHundler } logger.Root().SetHandler(hundler) } // Debugf is wrapper function func Debugf(format string, args ...interface{}) { logger.Debug(fmt.Sprintf(format, args...)) } // Infof is wrapper function func Infof(format string, args ...interface{}) { logger.Info(fmt.Sprintf(format, args...)) } // Warnf is wrapper function func Warnf(format string, args ...interface{}) { logger.Warn(fmt.Sprintf(format, args...)) } // Errorf is wrapper function func Errorf(format string, args ...interface{}) { logger.Error(fmt.Sprintf(format, args...)) } // Fatalf is wrapper function func Fatalf(format string, args ...interface{}) { logger.Crit(fmt.Sprintf(format, args...)) } go-cve-dictionary-0.3.1/main.go000066400000000000000000000020741337564334100163250ustar00rootroot00000000000000package main import ( "context" "flag" "fmt" "os" "strings" "github.com/google/subcommands" "github.com/kotakanbe/go-cve-dictionary/commands" ) // Name ... Name const Name string = "go-cve-dictionary" // Version ... Version var version = "0.2.0" // Revision of Git var revision string func main() { subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.FlagsCommand(), "") subcommands.Register(subcommands.CommandsCommand(), "") subcommands.Register(&commands.ServerCmd{}, "server") subcommands.Register(&commands.FetchJvnCmd{}, "fetchjvn") subcommands.Register(&commands.FetchNvdCmd{}, "fetchnvd") subcommands.Register(&commands.ListCmd{}, "list") var v = flag.Bool("v", false, "Show version") if envArgs := os.Getenv("GO_CVE_DICTIONARY_ARGS"); 0 < len(envArgs) { flag.CommandLine.Parse(strings.Fields(envArgs)) } else { flag.Parse() } if *v { fmt.Printf("go-cve-dictionary %s %s\n", version, revision) os.Exit(int(subcommands.ExitSuccess)) } ctx := context.Background() os.Exit(int(subcommands.Execute(ctx))) } go-cve-dictionary-0.3.1/models/000077500000000000000000000000001337564334100163325ustar00rootroot00000000000000go-cve-dictionary-0.3.1/models/models.go000066400000000000000000000202271337564334100201470ustar00rootroot00000000000000package models import ( "fmt" "strings" "time" "github.com/fatih/color" "github.com/jinzhu/gorm" ) // FeedMeta has meta information about fetched feeds type FeedMeta struct { gorm.Model `json:"-" xml:"-"` URL string Hash string LastModifiedDate string LatestHash string `json:"-" gorm:"-"` LatestLastModifiedDate string `json:"-" gorm:"-"` } // UpToDate checks whether last fetched feed is up to date func (f FeedMeta) UpToDate() bool { return !f.Newly() && f.Hash == f.LatestHash } // OutDated checks whether last fetched feed is out dated func (f FeedMeta) OutDated() bool { return !f.Newly() && f.Hash != f.LatestHash } // Newly checks whether not fetched yet func (f FeedMeta) Newly() bool { return f.Hash == "" } // StatusForStdout returns a status of fetched feed func (f FeedMeta) StatusForStdout() string { if f.Newly() { return "Newly" } else if f.OutDated() { red := color.New(color.FgRed, color.Bold).SprintFunc() return red("Out-Dated") } else if f.UpToDate() { return color.GreenString("Up-to-Date") } return "Unknown" } const ( nvdxml = "NVD(XML)" nvdjson = "NVD(JSON)" jvn = "JVN" ) func (f FeedMeta) color(str string) string { if f.OutDated() { return color.WhiteString(str) } else if f.UpToDate() { return color.HiBlackString(str) } return str } func (f FeedMeta) source() string { if strings.Contains(f.URL, "nvdcve-2.0-") { return nvdxml } else if strings.Contains(f.URL, "nvdcve-1.0-") { return nvdjson } else if strings.Contains(f.URL, "jvndb") { return jvn } return "Unknown" } // FetchOption returns a option of fetch subcommand for list subcommand func (f FeedMeta) FetchOption() string { switch f.source() { case nvdxml: return "fetchnvd -xml" case nvdjson: return "fetchnvd" case jvn: return "fetchjvn" default: return "" } } // Year returns year, whether xml or not of the feed func (f FeedMeta) Year() (year string, xml bool, err error) { switch f.source() { case nvdxml: return strings.TrimSuffix( strings.Split(f.URL, "nvdcve-2.0-")[1], ".xml.gz"), true, nil case nvdjson: return strings.TrimSuffix( strings.Split(f.URL, "nvdcve-1.0-")[1], ".json.gz"), false, nil case jvn: if strings.HasSuffix(f.URL, "jvndb.rdf") { return "modified", true, nil } else if strings.HasSuffix(f.URL, "jvndb_new.rdf") { return "recent", true, nil } else { return strings.TrimSuffix( strings.Split(f.URL, "jvndb_")[1], ".rdf"), true, nil } default: return "", false, fmt.Errorf("Failed to parse URL: %s", f.URL) } } func (f FeedMeta) modifiedTimesToStrs() (fetched, latest string) { switch f.source() { case nvdxml, nvdjson: layout := "2006-01-02T15:04:05-07:00" last, _ := time.Parse(layout, f.LastModifiedDate) latest, _ := time.Parse(layout, f.LatestLastModifiedDate) return last.Format("2006/1/2-15:04"), latest.Format("2006/1/2-15:04") case jvn: layout := "2006/01/02 15:04:05" last, _ := time.Parse(layout, f.LastModifiedDate) latest, _ := time.Parse(layout, f.LatestLastModifiedDate) return last.Format("2006/1/2-15:04"), latest.Format("2006/1/2-15:04") default: return "Unknown", "Unknown" } } // ToTableWriterRow generate data for table writer func (f FeedMeta) ToTableWriterRow() []string { y, _, _ := f.Year() fetched, latest := f.modifiedTimesToStrs() return []string{ f.color(f.source()), f.color(y), f.StatusForStdout(), f.color(fetched), f.color(latest), } } // CveDetail is a parent of Jnv/Nvd model type CveDetail struct { gorm.Model `json:"-" xml:"-"` CveID string NvdXML *NvdXML `json:",omitempty"` NvdJSON *NvdJSON `json:",omitempty"` Jvn *Jvn `json:",omitempty"` } // NvdXML is a model of NVD type NvdXML struct { gorm.Model `json:"-" xml:"-"` CveDetailID uint `json:"-" xml:"-"` CveID string Summary string `sql:"type:text"` Cvss2 Cvss2 Cpes []Cpe `json:",omitempty"` Cwes []Cwe References []Reference PublishedDate time.Time LastModifiedDate time.Time } // NvdJSON is a struct of NVD JSON // https://scap.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema type NvdJSON struct { gorm.Model `json:"-" xml:"-"` CveDetailID uint `json:"-" xml:"-"` // DataType string // DataFormat string // DataVersion string CveID string Descriptions []Description Cvss2 Cvss2Extra Cvss3 Cvss3 Cwes []Cwe Cpes []Cpe Affects []Affect References []Reference // Assigner string PublishedDate time.Time LastModifiedDate time.Time } // Jvn is a model of JVN type Jvn struct { gorm.Model `json:"-" xml:"-"` CveDetailID uint `json:"-" xml:"-"` CveID string Title string Summary string `sql:"type:text"` JvnLink string JvnID string Cvss2 Cvss2 Cvss3 Cvss3 Cpes []Cpe `json:",omitempty"` References []Reference PublishedDate time.Time LastModifiedDate time.Time } // Cwe has CweID type Cwe struct { gorm.Model `json:"-" xml:"-"` NvdXMLID uint `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` JvnID uint `json:"-" xml:"-"` CweID string } // Cpe is Child model of Jvn/Nvd. // see https://www.ipa.go.jp/security/vuln/CPE.html // In NVD JSON, // configurations>nodes>cpe>valunerable: true type Cpe struct { gorm.Model `json:"-" xml:"-"` JvnID uint `json:"-" xml:"-"` NvdXMLID uint `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` CpeBase EnvCpes []EnvCpe } // EnvCpe is a Environmental CPE // Only NVD JSON has this information. // configurations>nodes>cpe>valunerable: false type EnvCpe struct { gorm.Model `json:"-" xml:"-"` CpeID uint `json:"-" xml:"-"` CpeBase } // CpeBase has common args of Cpe and EnvCpe type CpeBase struct { URI string FormattedString string WellFormedName string `sql:"type:text"` CpeWFN VersionStartExcluding string VersionStartIncluding string VersionEndExcluding string VersionEndIncluding string } // CpeWFN has CPE Well Formed name informaiton type CpeWFN struct { Part string Vendor string Product string Version string Update string Edition string Language string SoftwareEdition string TargetSW string TargetHW string Other string } // Reference is Child model of Jvn/Nvd. // It holds reference information about the CVE. type Reference struct { gorm.Model `json:"-" xml:"-"` NvdXMLID uint `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` JvnID uint `json:"-" xml:"-"` Source string Link string `sql:"type:text"` } // Affect has vendor/product/version info in NVD JSON type Affect struct { gorm.Model `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` Vendor string Product string Version string } // Cvss3 has CVSS Version 3 info // NVD JSON and JVN has CVSS3 info type Cvss3 struct { gorm.Model `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` JVNID uint `json:"-" xml:"-"` VectorString string AttackVector string AttackComplexity string PrivilegesRequired string UserInteraction string Scope string ConfidentialityImpact string IntegrityImpact string AvailabilityImpact string BaseScore float64 BaseSeverity string ExploitabilityScore float64 ImpactScore float64 } // Cvss2 has CVSS Version 2 info type Cvss2 struct { gorm.Model `json:"-" xml:"-"` NvdXMLID uint `json:"-" xml:"-"` JvnID uint `json:"-" xml:"-"` VectorString string AccessVector string AccessComplexity string Authentication string ConfidentialityImpact string IntegrityImpact string AvailabilityImpact string BaseScore float64 // NVD JSON and JVN has severity (Not in NVD XML) Severity string } // Cvss2Extra has extra CVSS V2 info type Cvss2Extra struct { NvdJSONID uint `json:"-" xml:"-"` Cvss2 ExploitabilityScore float64 ImpactScore float64 ObtainAllPrivilege bool ObtainUserPrivilege bool ObtainOtherPrivilege bool UserInteractionRequired bool } // Description has description of the CVE type Description struct { gorm.Model `json:"-" xml:"-"` NvdJSONID uint `json:"-" xml:"-"` Lang string Value string `sql:"type:text"` } go-cve-dictionary-0.3.1/models/models_test.go000066400000000000000000000017601337564334100212070ustar00rootroot00000000000000package models import ( "testing" ) func TestFeedMeta(t *testing.T) { var tests = []struct { in FeedMeta uptodate bool outdated bool newly bool }{ { in: FeedMeta{ Hash: "", LatestHash: "aaa", }, uptodate: false, outdated: false, newly: true, }, { in: FeedMeta{ Hash: "abc", LatestHash: "def", }, uptodate: false, outdated: true, newly: false, }, { in: FeedMeta{ Hash: "def", LatestHash: "def", }, uptodate: true, outdated: false, newly: false, }, } for i, tt := range tests { aup := tt.in.UpToDate() if tt.uptodate != aup { t.Errorf("[%d] up expected: %#v\n actual: %#v\n", i, tt.uptodate, aup) } aout := tt.in.OutDated() if tt.outdated != aout { t.Errorf("[%d] out expected: %#v\n actual: %#v\n", i, tt.outdated, aout) } anew := tt.in.Newly() if tt.newly != anew { t.Errorf("[%d] newly expected: %#v\n actual: %#v\n", i, tt.newly, anew) } } } go-cve-dictionary-0.3.1/server/000077500000000000000000000000001337564334100163555ustar00rootroot00000000000000go-cve-dictionary-0.3.1/server/server.go000066400000000000000000000036171337564334100202210ustar00rootroot00000000000000package server import ( "fmt" "net/http" "os" "path/filepath" "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" log "github.com/kotakanbe/go-cve-dictionary/log" "github.com/labstack/echo" "github.com/labstack/echo/middleware" ) // Start starts CVE dictionary HTTP Server. func Start(logDir string, driver db.DB) error { e := echo.New() e.Debug = config.Conf.Debug // Middleware e.Use(middleware.Logger()) e.Use(middleware.Recover()) // setup access logger logPath := filepath.Join(logDir, "access.log") if _, err := os.Stat(logPath); os.IsNotExist(err) { if _, err := os.Create(logPath); err != nil { return err } } f, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY, 0600) if err != nil { return err } defer f.Close() e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ Output: f, })) // Routes e.GET("/health", health()) e.GET("/cves/:id", getCve(driver)) e.POST("/cpes", getCveByCpeName(driver)) bindURL := fmt.Sprintf("%s:%s", config.Conf.Bind, config.Conf.Port) log.Infof("Listening on %s", bindURL) e.Start(bindURL) return nil } // Handler func health() echo.HandlerFunc { return func(c echo.Context) error { return c.String(http.StatusOK, "") } } // Handler func getCve(driver db.DB) echo.HandlerFunc { return func(c echo.Context) error { cveid := c.Param("id") cveDetail, err := driver.Get(cveid) if err != nil { log.Errorf("%s", err) return err } return c.JSON(http.StatusOK, &cveDetail) } } type cpeName struct { Name string `form:"name"` } func getCveByCpeName(driver db.DB) echo.HandlerFunc { return func(c echo.Context) error { cpe := cpeName{} err := c.Bind(&cpe) if err != nil { log.Errorf("%s", err) return err } cveDetails, err := driver.GetByCpeURI(cpe.Name) if err != nil { log.Errorf("%s", err) return err } return c.JSON(http.StatusOK, &cveDetails) } } go-cve-dictionary-0.3.1/util/000077500000000000000000000000001337564334100160245ustar00rootroot00000000000000go-cve-dictionary-0.3.1/util/util.go000066400000000000000000000007721337564334100173360ustar00rootroot00000000000000package util import ( "os" "path/filepath" "runtime" ) // GenWorkers generate workders func GenWorkers(num int) chan<- func() { tasks := make(chan func()) for i := 0; i < num; i++ { go func() { for f := range tasks { f() } }() } return tasks } // GetDefaultLogDir returns default log directory func GetDefaultLogDir() string { defaultLogDir := "/var/log/vuls" if runtime.GOOS == "windows" { defaultLogDir = filepath.Join(os.Getenv("APPDATA"), "vuls") } return defaultLogDir }