pax_global_header00006660000000000000000000000064147367325000014521gustar00rootroot0000000000000052 comment=6e3f13f1f11cbc3566d145d8da94018eaad9628d certstream-server-go-1.7.0/000077500000000000000000000000001473673250000156065ustar00rootroot00000000000000certstream-server-go-1.7.0/.github/000077500000000000000000000000001473673250000171465ustar00rootroot00000000000000certstream-server-go-1.7.0/.github/goreleaser.yml000066400000000000000000000070311473673250000220220ustar00rootroot00000000000000project_name: certstream-server-go before: hooks: - go mod download builds: - main: ./cmd/certstream-server-go ldflags: -s -w -X github.com/d-Rickyy-b/certstream-server-go/internal/config.Version={{.Version}} env: - CGO_ENABLED=0 goos: - linux - darwin - windows goarch: - 386 - amd64 - arm - arm64 ignore: - goos: darwin goarch: 386 - goos: darwin goarch: arm - goos: windows goarch: arm - goos: windows goarch: arm64 - goos: windows goarch: 386 checksum: name_template: '{{.ProjectName}}_{{.Version}}_checksums.txt' changelog: skip: true dockers: - image_templates: - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' - '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:{{.Tag}}{{ end }}' - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' - '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest-amd64{{ end }}' goarch: amd64 use: buildx extra_files: - config.sample.yaml build_flag_templates: - "--pull" - "--label=org.opencontainers.image.title={{.ProjectName}}" - "--label=org.opencontainers.image.description=Certstream server written in Go" - "--label=org.opencontainers.image.created={{.Date}}" - "--label=org.opencontainers.image.source=https://github.com/d-Rickyy-b/certstream-server-go" - "--label=org.opencontainers.image.revision={{.FullCommit}}" - "--label=org.opencontainers.image.version={{.Version}}" - "--platform=linux/amd64" - image_templates: - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' - '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:latest-arm64{{ end }}' - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' - '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest-arm64{{ end }}' goarch: arm64 use: buildx extra_files: - config.sample.yaml build_flag_templates: - "--pull" - "--label=org.opencontainers.image.title={{.ProjectName}}" - "--label=org.opencontainers.image.description=Certstream server written in Go" - "--label=org.opencontainers.image.created={{.Date}}" - "--label=org.opencontainers.image.source=https://github.com/d-Rickyy-b/certstream-server-go" - "--label=org.opencontainers.image.revision={{.FullCommit}}" - "--label=org.opencontainers.image.version={{.Version}}" - "--platform=linux/arm64" docker_manifests: - name_template: '0rickyy0/{{.ProjectName}}:{{.Tag}}' image_templates: - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' - name_template: 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}' image_templates: - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' - name_template: '{{ if not .Prerelease }}0rickyy0/{{.ProjectName}}:latest{{ end }}' image_templates: - '0rickyy0/{{.ProjectName}}:{{.Tag}}-amd64' - '0rickyy0/{{.ProjectName}}:{{.Tag}}-arm64' - name_template: '{{ if not .Prerelease }}ghcr.io/d-rickyy-b/{{.ProjectName}}:latest{{ end }}' image_templates: - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-amd64' - 'ghcr.io/d-rickyy-b/{{.ProjectName}}:{{.Tag}}-arm64' archives: - format: binary name_template: >- {{- .ProjectName }}_ {{- .Version}}_ {{- if eq .Os "darwin" }}macOS{{- else }}{{ .Os }}{{ end }}_ {{- if eq .Arch "386" }}i386{{- else }}{{ .Arch }}{{ end }} certstream-server-go-1.7.0/.github/workflows/000077500000000000000000000000001473673250000212035ustar00rootroot00000000000000certstream-server-go-1.7.0/.github/workflows/changelog_reminder.yml000066400000000000000000000007731473673250000255510ustar00rootroot00000000000000name: Changelog Reminder on: pull_request permissions: pull-requests: write jobs: remind: name: Changelog Reminder runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Changelog Reminder uses: mskelton/changelog-reminder-action@v3 with: message: "@${{ github.actor }} We couldn't find any modification to the CHANGELOG.md file. If your changes are not suitable for the changelog, that's fine. Otherwise please add them to the changelog!" certstream-server-go-1.7.0/.github/workflows/codeql-analysis.yml000066400000000000000000000052471473673250000250260ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above branches: [ "master" ] schedule: - cron: '43 9 * * 0' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'go' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 certstream-server-go-1.7.0/.github/workflows/release_build.yml000066400000000000000000000025641473673250000245340ustar00rootroot00000000000000name: build on: push: tags: - "*" jobs: build: name: Release build runs-on: ubuntu-latest permissions: packages: write contents: write steps: - name: Set up Go 1.22 uses: actions/setup-go@v5 with: go-version: ^1.22 id: go - name: Check out code into the Go module directory uses: actions/checkout@v4 with: fetch-depth: 0 # See: https://goreleaser.com/ci/actions/ - name: Setup QEMU # Used for cross-compiling with goreleaser / docker uses: docker/setup-qemu-action@v3 - name: Setup Docker Buildx # Used for cross-compiling with goreleaser / docker uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 with: version: latest args: release --clean --config .github/goreleaser.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} certstream-server-go-1.7.0/.gitignore000066400000000000000000000006211473673250000175750ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib /certstream-server-go # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work .idea .cache *.pprof dist/ # Ignore actual config files config.yaml config.yml certstream-server-go-1.7.0/.golangci.yml000066400000000000000000000056301473673250000201760ustar00rootroot00000000000000# Options for analysis running. run: # The default concurrency value is the number of available CPU. concurrency: 4 # Timeout for analysis, e.g. 30s, 5m. # Default: 1m timeout: 1m # Include test files or not. # Default: true tests: false # Which files to skip: they will be analyzed, but issues from them won't be reported. # Default value is empty list, but there is no need to include all autogenerated files, # we confidently recognize autogenerated files. #skip-files: # - ".*\\.my\\.go$" # - lib/bad.go # Allowed values: readonly|vendor|mod # By default, it isn't set. modules-download-mode: readonly # Allow multiple parallel golangci-lint instances running. # If false (default) - golangci-lint acquires file lock on start. allow-parallel-runners: false linters: # Run only fast linters from enabled linters set (first run won't be fast) # Default: false # fast: true # Disable all linters. disable-all: true # Enable specific linter # https://golangci-lint.run/usage/linters/#enabled-by-default enable: - bodyclose - containedctx # - deadcode - depguard - dogsled - dupl - dupword - durationcheck - errcheck - errchkjson - errname - errorlint - execinquery - exhaustive - exportloopref - gochecknoinits - gocritic - godot # - godox - gofmt # - gofumpt - goimports # - golint # - gomnd - gomoddirectives - gomodguard - goprintffuncname - gosec - gosimple - govet - grouper - importas - ineffassign - ireturn - lll - loggercheck - maintidx - makezero - misspell - nakedret - nestif - nilerr - nilnil - nlreturn - nolintlint - nosprintfhostport - paralleltest - prealloc - predeclared - promlinter - reassign - revive - staticcheck - stylecheck - tenv - typecheck - unconvert - unparam - unused - usestdlibvars - varnamelen - wastedassign - whitespace - wrapcheck - wsl linters-settings: nlreturn: # Size of the block (including return statement that is still "OK") so no return split required. # Default: 1 block-size: 2 wsl: enforce-err-cuddling: true allow-cuddle-declarations: true allow-assign-and-call: true allow-cuddle-with-calls: - log.Println - log.Printf - RLock - RUnlock - Lock - Unlock allow-assign-and-anything: true gofumpt: extra-rules: true lll: line-length: 160 varnamelen: ignore-decls: - w io.Writer - w io.WriteCloser - w http.ResponseWriter - r *http.Request - r chi.Router - r *chi.Mux - i int issues: # List of regexps of issue texts to exclude. exclude-rules: # Exclude some linters from running on tests files. - path: _test\.go linters: - nlreturn certstream-server-go-1.7.0/CHANGELOG.md000066400000000000000000000105661473673250000174270ustar00rootroot00000000000000# Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added - Support for websocket compression - disabled by default (#40) - Support for non-browsers by implementing server initiated heartbeats (#39) - Start new ct-watchers as new ct logs become available (#42) ### Changed ### Fixed - Fixed a possible race condition when accessing metrics ### Docs ## [1.6.0] - 2024-03-05 ### Added - New metric for skipped certs per client (#34) ## [1.5.2] - 2024-02-17 ### Fixed - Fixed an issue with ip whitelists for the websocket server (#33) ## [1.5.1] - 2024-01-18 ### Fixed - Fixed a rare issue where it was possible for the all_domains json property (or data property in case of the domains-only endpoint) to be null ## [1.5.0] - 2023-12-21 ### Added - New `-version` switch to print version and exit afterwards - Print version on every run of the tool - Count and log number of skipped certificates per client ### Changed - Update to chi/v5 - Update ct-watcher timeout from 5 to 30 seconds ### Fixed - Prevent invalid subscription types to be used - Kill connection after broadcasthandler was stopped ## [1.4.0] - 2023-11-29 ### Added - Config option to use X-Forwarded-For or X-Real-IP header as client IP - Config option to whitelist client IPs for both websocket and metrics endpoints - Config option to enable system metrics (cpu, memory, etc.) ## [1.3.2] - 2023-11-28 ### Fixed - Memory leak related to clients disconnecting from the websocket not being handled properly ## [1.3.1] - 2023-09-18 ### Changed - Updated config.sample.yaml to run both certstream and prometheus metrics on same socket ### Docs - Fixed wrong docker command in readme ## [1.3.0] - 2023-04-11 ### Added - Calculate and display Sha256 sum of certificate ### Changed - Update dependencies - Better logging for CT log errors ### Fixed - End execution after all workers stopped - Implement timeout for the http client - Keep ct watcher from crashing upon a connection reset from server ## [1.2.2] - 2023-01-10 ### Added - Two docker-compose files - Check for presence of .yml or .yaml files in the current directory ### Fixed - Handle sudden disconnects of CT logs ### Docs - Added [wiki entry for docker-compose](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics) ## [1.2.1] - 2022-12-16 ### Changed - Updated ci pipeline to use new setup-go and checkout actions - Use correct package name `github.com/d-Rickyy-b/certstream-server-go` ## [1.2.0] - 2022-12-15 ### Added - Log x-Forwarded-For header for requests - More logging for certain error situations - Add operator to ct log cert count metrics ### Changed - Updated certificate-transparency-go dependency to v1.1.4 - Code improvements, adhering to styleguide - Rename module to certstream-server-go - Use log_list.json instead of all_logs_list.json ## [1.1.0] - 2022-10-19 Fix for missing loglist urls. ### Fixed Fixed the connection issue due to the offline Google loglist urls. ## [1.0.0] - 2022-08-08 Initial release! First stable version of certstream-server-go is published as v1.0.0 [unreleased]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.6.0...HEAD [1.6.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.2...v1.6.0 [1.5.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.4.0...v1.5.0 [1.4.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.2...v1.4.0 [1.3.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.1...v1.3.2 [1.3.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.3.0...v1.3.1 [1.3.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.2...v1.3.0 [1.2.2]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.1...v1.2.2 [1.2.1]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.2.0...v1.2.1 [1.2.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/d-Rickyy-b/certstream-server-go/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/d-Rickyy-b/certstream-server-go/tree/v1.0.0 certstream-server-go-1.7.0/Dockerfile000066400000000000000000000007671473673250000176120ustar00rootroot00000000000000FROM alpine WORKDIR /app ENV USER=certstreamserver ENV UID=10001 # Create user RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ --shell "/sbin/nologin" \ --no-create-home \ --uid "${UID}" \ "${USER}" # Copy our static executable. COPY certstream-server-go /app/certstream-server-go COPY ./config.sample.yaml /app/config.yaml # Use an unprivileged user. USER certstreamserver:certstreamserver EXPOSE 8080 ENTRYPOINT ["/app/certstream-server-go"]certstream-server-go-1.7.0/Dockerfile_multistage000066400000000000000000000026361473673250000220450ustar00rootroot00000000000000# Thanks to https://chemidy.medium.com/create-the-smallest-and-secured-golang-docker-image-based-on-scratch-4752223b7324 ############################ # STEP 1 build executable binary ############################ FROM golang:alpine AS builder ENV USER=certstreamserver ENV UID=10001 # Create user RUN adduser \ --disabled-password \ --gecos "" \ --home "/nonexistent" \ --shell "/sbin/nologin" \ --no-create-home \ --uid "${UID}" \ "${USER}" # Install git. Git is required for fetching the dependencies. RUN apk update && apk add --no-cache git WORKDIR $GOPATH/src/certstream-server-go/ COPY . . # Fetch dependencies. RUN go mod download # Build the binary. RUN go build -ldflags="-w -s" -o /go/bin/certstream-server-go $GOPATH/src/certstream-server-go/cmd/certstream-server-go/ RUN chown -R "${USER}:${USER}" /go/bin/certstream-server-go ############################ # STEP 2 build a small image ############################ FROM alpine WORKDIR /app # Import the user and group files from the builder. COPY --from=builder /etc/passwd /etc/passwd COPY --from=builder /etc/group /etc/group # Copy our static executable. COPY --from=builder /go/bin/certstream-server-go /app/certstream-server-go COPY --chown=certstreamserver:certstreamserver ./config.sample.yaml /app/config.yaml # Use an unprivileged user. USER certstreamserver:certstreamserver EXPOSE 8080 ENTRYPOINT ["/app/certstream-server-go"] certstream-server-go-1.7.0/LICENSE000066400000000000000000000020531473673250000166130ustar00rootroot00000000000000MIT License Copyright (c) 2022 d-Rickyy-b Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. certstream-server-go-1.7.0/README.md000066400000000000000000000165751473673250000171030ustar00rootroot00000000000000![certstream-server-go logo](https://github.com/d-Rickyy-b/certstream-server-go/blob/master/docs/img/certstream-server-go_logo.png?raw=true) # Certstream Server Go [![build](https://github.com/d-Rickyy-b/certstream-server-go/actions/workflows/release_build.yml/badge.svg)](https://github.com/d-Rickyy-b/certstream-server-go/actions/workflows/release_build.yml) [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/0rickyy0/certstream-server-go?label=docker&sort=semver)](https://hub.docker.com/repository/docker/0rickyy0/certstream-server-go) [![Go Reference](https://pkg.go.dev/badge/github.com/d-Rickyy-b/certstream-server-go.svg)](https://pkg.go.dev/github.com/d-Rickyy-b/certstream-server-go) This project aims to be a drop-in replacement for the [certstream server](https://github.com/CaliDog/certstream-server/) by Calidog. This tool aggregates, parses, and streams certificate data from multiple [certificate transparency logs](https://www.certificate-transparency.org/what-is-ct) via websocket connections to the clients. Everyone can use this project to analyze newly created TLS certificates as they are issued. ## Motivation From the moment I first found out about the certificate transparency logs, I was absolutely amazed by the great software of [Calidog](https://github.com/CaliDog/), which made the transparency log easier accessible for everyone. Their software "Certstream" parses the log and provides it in an easy-to-use format: json. After creating my first application that utilized the certstream server, I found that the hosted (demo) version of the server wasn't as reliable as I thought it would be. I got disconnects and sometimes other errors. Eventually the provided server was still only thought to be **a demo**. I quickly thought about running my own instance of certstream. But I didn't want to install Elixir/Erlang on my server. Sure, I could have used Docker, but on second thought, I was really into the idea of creating an alternative server written in Go. "Why Go?", you might ask. Because it is a great language that compiles to native binaries on all major architectures and OSes. All the cool kids are using it right now. ## Getting started Setting up an instance of the certstream server is simple. You can either download and compile the code yourself, or use one of the [precompiled binaries](https://github.com/d-Rickyy-b/certstream-server-go/releases). ### Docker There's also a prebuilt [Docker image](https://hub.docker.com/repository/docker/0rickyy0/certstream-server-go) available. You can use it by running this command: `docker run -d -v /path/to/config.yaml:/app/config.yaml -p 8080:8080 0rickyy0/certstream-server-go` > ⚠️ If you don't mount your own config file, the default config (config.sample.yaml) will be used. For more details, check out the [wiki](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Configuration). ## Connecting certstream-server-go offers multiple endpoints to connect to. | Config | Default | Function | |--------------------|-----------------|-------------------------------------------------------------------------------------------| | `full_url` | `/full-stream` | Constant stream of new certificates with all details available | | `lite_url` | `/` | Constant stream of new certificates with reduced details (no `as_der` and `chain` fields) | | `domains_only_url` | `/domains-only` | Constant stream of domains found in new certificates | You can connect to the certstream-server by opening a **websocket connection** to any of the aforementioned endpoints. After you're connected, certificate information will be streamed to your websocket. The server requires you to send a **ping message** at least every 60 seconds (it's recommended to use an interval of 30s for pings). If the server does not receive a ping message for more than this time, it will disconnect you. The server will **not** send out ping messages to your client. Read more about ping/pong WebSocket messages in the [Mozilla Developer Docs](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers#pings_and_pongs_the_heartbeat_of_websockets). ### Performance At idle (no clients connected), the server uses about **40 MB** of RAM, **14.5 Mbit/s** and **4-10% CPU** (Oracle Free Tier) on average while processing around **250-300 certificates per second**. ### Monitoring **certstream-server-go** also offers a Prometheus metrics endpoint at `/metrics`. You can use this to monitor the server with Prometheus and Grafana. For an in-depth guide on how to do this, please refer to the [wiki](https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics). ![grafana dashboard](https://user-images.githubusercontent.com/5798157/211434271-4350766d-2942-4fcb-8fda-f131f3f61cea.png) ### Example To receive a live example for any of the endpoints, just send an HTTP GET request to the endpoints with `/example.json` appended to the endpoint. For example: `/full-stream/example.json`. This shows the lite format of a certificate update. ```json { "data": { "cert_index": 712420366, "cert_link": "https://yeti2022-2.ct.digicert.com/log/ct/v1/get-entries?start=712420366&end=712420366", "leaf_cert": { "all_domains": [ "cmslieferhit.e06.k-k.de" ], "extensions": { "authorityInfoAccess": "URI:http://r3.i.lencr.org/, URI:http://r3.o.lencr.org", "authorityKeyIdentifier": "keyid:14:2e:b3:17:b7:58:56:cb:ae:50:09:40:e6:1f:af:9d:8b:14:c2:c6", "basicConstraints": "CA:FALSE", "keyUsage": "Digital Signature, Key Encipherment", "subjectAltName": "DNS:cmslieferhit.e06.k-k.de", "subjectKeyIdentifier": "keyid:4e:cb:ae:47:84:a8:92:f7:e7:de:78:d1:00:9e:d9:cc:80:ac:0b:ce" }, "fingerprint": "27:58:3D:01:3D:71:B8:D3:A6:6E:2C:7A:86:3A:E9:1F:DB:F0:1B:5D", "sha1": "27:58:3D:01:3D:71:B8:D3:A6:6E:2C:7A:86:3A:E9:1F:DB:F0:1B:5D", "sha256": "57:61:38:C0:3C:03:A3:34:6A:0B:32:89:11:1B:74:AB:8A:DF:A5:02:9F:06:43:E6:F3:0E:69:F3:0E:4E:4E:FC", "not_after": 1667028404, "not_before": 1659252405, "serial_number": "0498BDF812FAF923FEBD5EF7B374899FC61A", "signature_algorithm": "sha256, rsa", "subject": { "C": null, "CN": "cmslieferhit.e06.k-k.de", "L": null, "O": null, "OU": null, "ST": null, "aggregated": "/CN=cmslieferhit.e06.k-k.de", "email_address": null }, "issuer": { "C": "US", "CN": "R3", "L": null, "O": "Let's Encrypt", "OU": null, "ST": null, "aggregated": "/C=US/CN=R3/O=Let's Encrypt", "email_address": null }, "is_ca": false }, "seen": 1659301203.904, "source": { "name": "DigiCert Yeti2022-2 Log", "url": "https://yeti2022-2.ct.digicert.com/log" }, "update_type": "PrecertLogEntry" }, "message_type": "certificate_update" } ``` certstream-server-go-1.7.0/cmd/000077500000000000000000000000001473673250000163515ustar00rootroot00000000000000certstream-server-go-1.7.0/cmd/certstream-server-go/000077500000000000000000000000001473673250000224315ustar00rootroot00000000000000certstream-server-go-1.7.0/cmd/certstream-server-go/main.go000066400000000000000000000041521473673250000237060ustar00rootroot00000000000000package main import ( "flag" "fmt" "log" "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/d-Rickyy-b/certstream-server-go/internal/metrics" "github.com/d-Rickyy-b/certstream-server-go/internal/web" ) // main is the entry point for the application. func main() { configFile := flag.String("config", "config.yml", "path to the config file") versionFlag := flag.Bool("version", false, "Print the version and exit") flag.Parse() if *versionFlag { fmt.Printf("certstream-server-go v%s\n", config.Version) return } log.SetFlags(log.LstdFlags | log.Lshortfile) log.Printf("Starting certstream-server-go v%s\n", config.Version) conf, err := config.ReadConfig(*configFile) if err != nil { log.Fatalln("Error while parsing yaml file:", err) } webserver := web.NewWebsocketServer(conf.Webserver.ListenAddr, conf.Webserver.ListenPort, conf.Webserver.CertPath, conf.Webserver.CertKeyPath) setupMetrics(conf, webserver) go webserver.Start() watcher := certificatetransparency.Watcher{} watcher.Start() } // setupMetrics configures the webserver to handle prometheus metrics according to the config. func setupMetrics(conf config.Config, webserver *web.WebServer) { if conf.Prometheus.Enabled { // If prometheus is enabled, and interface is either unconfigured or same as webserver config, use existing webserver if (conf.Prometheus.ListenAddr == "" || conf.Prometheus.ListenAddr == conf.Webserver.ListenAddr) && (conf.Prometheus.ListenPort == 0 || conf.Prometheus.ListenPort == conf.Webserver.ListenPort) { log.Println("Starting prometheus server on same interface as webserver") webserver.RegisterPrometheus(conf.Prometheus.MetricsURL, metrics.WritePrometheus) } else { log.Println("Starting prometheus server on new interface") metricsServer := web.NewMetricsServer(conf.Prometheus.ListenAddr, conf.Prometheus.ListenPort, conf.Prometheus.CertPath, conf.Prometheus.CertKeyPath) metricsServer.RegisterPrometheus(conf.Prometheus.MetricsURL, metrics.WritePrometheus) go metricsServer.Start() } } } certstream-server-go-1.7.0/config.sample.yaml000066400000000000000000000005751473673250000212260ustar00rootroot00000000000000webserver: listen_addr: "0.0.0.0" listen_port: 8080 full_url: "/full-stream" lite_url: "/" domains_only_url: "/domains-only" cert_path: "" cert_key_path: "" compression_enabled: false prometheus: enabled: true listen_addr: "0.0.0.0" listen_port: 8080 metrics_url: "/metrics" expose_system_metrics: false real_ip: false whitelist: - "127.0.0.1/8" certstream-server-go-1.7.0/docker/000077500000000000000000000000001473673250000170555ustar00rootroot00000000000000certstream-server-go-1.7.0/docker/docker-compose.metrics.yml000066400000000000000000000044001473673250000241550ustar00rootroot00000000000000version: '2' # Make sure to create the sub directories "prometheus", "prometheus_data", "grafana", "grafana_data" and "certstream" # and create the config files for all three services. For further details please refer to https://github.com/d-Rickyy-b/certstream-server-go/wiki/Collecting-and-Visualizing-Metrics networks: monitoring: driver: bridge ipam: config: - subnet: 172.90.0.0/24 gateway: 172.90.0.1 services: prometheus: image: prom/prometheus:v2.40.5 restart: always # Configure the service to run as specific user. # user: "1000:1000" volumes: - ./prometheus/:/etc/prometheus/ - ./prometheus_data:/prometheus/ command: - '--config.file=/etc/prometheus/prometheus.yml' - '--storage.tsdb.path=/prometheus' - '--storage.tsdb.retention.time=1y' - '--web.console.libraries=/etc/prometheus/console_libraries' - '--web.console.templates=/etc/prometheus/consoles' - '--web.enable-lifecycle' ports: # Exposing Prometheus is NOT required, if you don't want to access it from outside the Docker network. # Using localhost enables you to use a reverse proxy (e.g. with basic auth) to access Prometheus in a more secure way. - 127.0.0.1:9090:9090 networks: - monitoring extra_hosts: - "host.docker.internal:host-gateway" grafana: image: grafana/grafana:9.3.1 restart: always # Configure the service to run as specific user. # user: "1000:1000" depends_on: - prometheus ports: - 127.0.0.1:8082:3000 volumes: - ./grafana_data:/var/lib/grafana - ./grafana/provisioning/:/etc/grafana/provisioning/ env_file: # changes to the grafana env file require a rebuild of the container. - ./grafana/config.monitoring networks: - monitoring certstream: image: 0rickyy0/certstream-server-go:latest restart: always # Configure the service to run as specific user. # user: "1000:1000" ports: - 127.0.0.1:8080:80 # Don't forget to open the other port in case you run the Prometheus endpoint on another port than the websocket server. # - 127.0.0.1:8081:81 volumes: - ./certstream/config.yml:/app/config.yml networks: - monitoring certstream-server-go-1.7.0/docker/docker-compose.yml000066400000000000000000000007241473673250000225150ustar00rootroot00000000000000version: '2' services: certstream: image: 0rickyy0/certstream-server-go:latest restart: always # Configure the service to run as specific user # user: "1000:1000" ports: - 127.0.0.1:8080:80 # Don't forget to open the other port in case you run the Prometheus endpoint on another port than the websocket server. # - 127.0.0.1:8081:81 volumes: - ./certstream/config.yml:/app/config.yml networks: - monitoring certstream-server-go-1.7.0/docs/000077500000000000000000000000001473673250000165365ustar00rootroot00000000000000certstream-server-go-1.7.0/docs/img/000077500000000000000000000000001473673250000173125ustar00rootroot00000000000000certstream-server-go-1.7.0/docs/img/certstream-server-go_logo.png000066400000000000000000000575451473673250000251400ustar00rootroot00000000000000PNG  IHDRR]ZiCCPsRGB IEC61966-2.1(uKA$JD# `,,4 jD$gB][6 */VERmT9#D2;vg؝k$d>dZ(wͻ頕a|QEWc 3^yZk Kq]Kjyqɕjp.  hrAkSd?L"X]_JJqg>KLXbx;:!q1(ًzeEirȬRDc$)Zq 2(Ww0^%x7YۃWKͽ 589h-8];5E%5#hK_(g[W]tyh4ަ pHYs   IDATxwTeea.MDD@4JD7"Q@P@E5 bB HtiR~ RBﰰl8)InrszݗL<9<DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDdi:n19OBޤxq`n5u#""2\t""]f`.Qg 1z9{P(""RҨP("""""2B(!JEDDDDDF%""""""#@B PDDDDDdP("""""2B(!JEDDDDDF%""""""#@B PDDDDDdP("""""2B(!JEDDDDDF%""""""#@B PDDDDDdP("""""2B(!5H;xA2X00Xgn }{n ǀ0r$ $pC4jc YX "?6UHw rbќFHwG5h`Ig͂dVMo*:kaIhhYϚG8zzcZXx xxՊ Lz@.^>|\/ pl`:ft|p=ppYSvp^,lIhff1~N>,Lз9OV,з.w#3L? .]}&~ X:gXss7o^A*g<-{aQb`nX3q[ 5+ uP l?S8zIY 0??[qs1,91&!Z_+`n..vXq'a \cѿs?0aӬ8z-o9\X^={ͮ~b.=\,0WVCM?] \nQcE@v Y 6pG xS ݶ/H6Bٗ qjc G 9~/.{rLLox`{o ̦.T8?bd7? ea{f?:o?pzdr\ !P&UCރ}c^  /4X d:1uܻ1Yqq3'R] gYwǦ{+p"pG8g3 ӁrƐo+hgO {p6re{Z9NYl>Z긧'Xq,]0Ii^kf}7Rnz!@6 2hOz}yAW[o'//HVN>IDaQ8 +I/dy݅}0I[s1N@X!B긋c識 f)`C,X/HL'q1N;Z5&k{A/H$Kdt`c\جxAɺS WbI&_w.~֒wg rJw}hof0uܞq?9j>ܚ:n:[긣R3eTecfg<:ڼdk7uѩYcKZOZ8 %]x ʿkci"Â@j^L Okdy ɊU40K1S5S8ʊ\Z'AK:E긻12VLK5= ::0,mUvf ٙy+-~~'6\-IXl {pS\E5 3pQ6B>,sPӾv$@Z^lIV$46rK "wLqvTF/HkSy^L6W# Lw o߽祎D} do3k7u0-v6ImƌDdlLwpf e;lyP`Ҏ85[_-ՔJWd7f׼JTc1|z98l]}6E `,o%988`[-_nIk&פjp gT{QT9Rݶ/d¬=m0E@:^l YTk<{ABqV 4Ң yw&:+Dr bQ-]nΦQŀSͻ}Ja Uj "1jkw]go%KJSAS]3Ӌ3lr}HJWF.d$^l Iu[9D, l~kз[ek`xg]6]`mNS]O_$zZQSx;TYԨF01lp:>4]ji;%5 Y$ѲyAfz'%%$XEWoVRGډ{I{,:nZoa UjqT]|ch̶Iw BӒ2oc7>}00ʣ]Ej1,/KzofO1l "f0I j3t1ˋgiT"EC&{A}k Ua!`W`Uy<^Ln_>+Z_G0/):f*f2`q'a >̵z/}xew>ע)y{j檘-o%c{GVr^OǬ%K٫3evZ1:GBH X(hzdsT~xx <6|uԍDb'u[iq'Zq|EPN~  зgw$ c*L_ԩ(&f17-i fk1E*CYF q_3x<+{ݙ}#`sr̞ly. b*8yqXgQ]0yܩ}:f:P&'2{Q^jZqT:]) G/p叩㮂Ip?WL%8zD}O} ~\;{h0@23d*3gItߧ\c~?31=Dw*[[xp(}"FS&yAban:VvI~)M0srU۟2S?a~/bF>Hi1o]0ΡY3&d3B'o _1x;:{G긯cFrG>ʛN$A(%uJV56J:{sΗކ~v(plhQXlߧ1[\HJ mȬ݊[qtWPmN( Xq4+u+1݊zƊ5Ŵzt"%ұ )כo*b&kSl,"N> f+0wBNK;DLSpG~ъgg+o{ ؿ?RǝJ[*ڷ511Q0W&S /pV]U02N\2*SX%[ iKF[1D5%ԚSؙ?gLJ=䍸$KcJC- WbK)SdHr7QlqD''XD5KڎEQN(ڵq'.&SOHюnǨko!6:݊[jljQ{ԖI(N6道)VO} q6arV4\PL̵Pa Ɗ# ^XPd|3sk_~Ӣ䛢 o+l%l6>\@+ak8G;r{t %.p8~.S}~E̺*x0fW %6vIC}z-Y/qo~\-#*qIyF/HPibՄ6/#6 })Ί9^ W8Ͳ+TdR(lBc }*;}Z3J5 IĬ#lW4{=Ksk0UU*+s f$U^%G+I긣9}.yȳr ! @qpZ6< yubL"]I tlīH9VImt+pqGsmLc'{feĭB~+wp.fp5gmwTg:0&P:$WMDam׎m̍. }}*- rhj긛`FE0UCMkVuƒnQӼ?-3w^E&6̕tcZJS 4r L Ӏ0ш, 'f^{V긞Fsig2I##b;ˊͻN5Un@cZJS=GnŊs4N$-3ۮ%g'[#ߥ w;6tj%@hq[q48TK;ݼ:1&P:RozA2ruӢh6IVڿܥͬ8 rqVǥ{GåpTx3fEVцL\e.{0P:k_۲$=:Ud4U0? 8;R@JȪ,^ \:.l)2?pt( V%m误g 1ڟ~P:ًObLw&= G^3 ؗtl?Lw)`WpL">:VUU}X.c>_(uCnEu)E t5ȗ8| 8ֈʻĘѾBnt!+^Ĭ< 8 3E4o1ΘBBnp6p5fVu`f9w*uܱ: uŀO/V#.c51$O+$21UEao_Oʰu+VSSHRΥO2Vmkхa/Sc8.`?4(U 4As`FacU&`Q:n]PdqߵO8yt38㗐Vw5l:u!"R%%ұBߞuJ=9N8 B߾ }/HV-pZM%n"7Ҁq7>>a8 NëcPZdѴqv(pu m^Ɵ1SM |`R~ ,PW""US(`{`\ow&g? E1SB~[9i0)>8}.X0pS%Xq|긷<囩kQXSHbÊt NS|; Y y('c 5F/H&dk/Hd*ړ%X9/ro:neSǝ:@b_UAdSZu""uS(-970k>^| Y _S YQS{)$j/H\z$ Ҟ KyȘDB~ Q( Z?l{A2 x=k̘o`nr~'bn(Nݨ{uLuYY=^Dq y  џɿ~,@7#6qܦUSdNwaFf71`oYć1 -Sy#ҴXqtO긇Sn}/{;?i `К!}:8Jđ0ٱ70%uG0^͘Y ؂|~.{ e2o`~lrD2{ Z`5$g@1F0CW؟HeJ79mJĩ&or1xk^= \/Hvl<5 IDATb/󎔎~#o_Y2|e*YTLb4=j8=uĊ[l}8:vβ_Di t 0OKOxlZj)oOk8 GP(ȈPNO`=)҄̔K+} & J7/ 7t/mV=YzSá,Y]lau7-m:(C{hnm,8@ʉms.%Ӏ/}UCKqVaV@҂>nQ8>Tg?C8SVDc)m9wYÔnIۏbF}[b}<دDo*O8:M}IŬ8J8 ЊљGaf,~KDDPZ۳B>_\;0758MB~ N7iZo澤bVͱh`;LyMXc&V 6 [ZqbGS8F;Fֳ(MnPзV^Dpna6"*зg'}f ~Vaob>-B~8Wُ*nn`}+~Ua1DV)ŌNh/8jjZ;VlZdž<jh_DPзggxAr>f׀arϻg^^ob'G]pӀzA;``7`1C2{ $/aFrήj䲏Y?*f3LYYI˟V=t{b-^mx Sة*EZqCy}ˊ]R]sl|r."+:ť^p;b&T 8ǼBuǴ}%1S<^mc_"4H +8L̍{fq{C?3o+7}4e5MBzxw٘$lV〇KB膥> RW3-z7{Sտq{0@666/,$pw+ӰR UB6f߀C~q 0vC`9 u|pqkSSoO-8KƘBr,зE/Hoc<3u{A0z^zw1pyۯ)ŀ4bK^,L\`}7Dda.Ut081KL`%``M cJ^{8|4AL/HN}ZG6·-9'W ec y8 8=uĚpaYg$c0yI*-p _kbu1s]{A{؂ߏ>f{Ar=зo%w6#{Xu[D;%"].ùaFpj] lL(,+^,IFlf pз&eS\kW 9:NBk%83?anX62pfFC+Fc9^\yO>bu{=/|&ziHJwqD,ܔ:p,dp;ffo^[yAOE 1CU$}\\'TM|$cFUNct/HF{A'0f4o^;zAIs=Onj~gP(҅RMjY@Mw'Z54?$F5*|^n7~| UjAo q0ӳWіnD̃Ӏekj>'^ 85i@_ Ig6P(eR0pՏtŀRݪ@ 񘑳~5{U"u aeL贱ϼL :xA6p7fi-9vI7bM'<8bzuF^쁩;o1vX6InT&k.L Hw93!v괨JyA p@qZM>\J 00g^.,і6d_LU pIdz^Xv\WD2JED6q0b*6@Y+ȷnKdbm"a2fuy11eW 31?0L[S40VTܑ1wз/k!,}=KWКZeۉ\ VmL0zb9z &KV){_qL{fu{-?WKt">JECe]R=ЊMR"U? }y J: o?=1ObG Y8s\^1^У@ qIxз_ Z 'gzC$i?_;Ĺ{dlFǵ69-{hoOǀ f0?ϼLK-;;`550c$  qܭ/8B:tq@긫m:.Gj::xA{k9_;i+oз޶Ga\_4ɦ'߅_sr `b<vU 9I7C߮lm$] }U?O,6&S٘u'=` -rmW<'Ovd 07ɻ*z~?&|8' џoodw[/H>u%:S }=6'Tb0Eb /f/1ej STD:XOJT{;%#p[긝Ayw<~, ^ɚ5ז< } {A$xA7{`m/HV }cWoc*Zu$,{A潙dz=.Td4"z Y G[0[Tp1I#"?Y〿5p{긛Xq4LNcnW(xhL!] dF0/K"> kl0/cw1klT&&ɻ}tnP C߾EQ=(k2.t ydOlOc6~ YޱV^c^I}5t%"m: t{G4PB~ SrZ3nO/H}oeɦi~<;gbWҾuobno-o" u$ }!&Gn= s1x!xqVԆc4T}~ ,t>?O7oF}%FLX 3 P%_q!/H~w> 's1NL }{зooJo6Cno1 1x;B8ME)pG x0ߛi[ }8?دpJEW4gX}d_3-Ni93| s|S۳kGr }m/H/|;UBbl@=)BM S6*/H| ~kMFv#\F̈%ݴ]r>8.d1`sRo}6; bHP(R@rv ` y^\~9O 19$]Ju^-xs|NMĥAo`mf$1z8 Vkѥ>8X1D(ߌ\:{-Bߞɷ1CqJfjP=ͰVv5\LwwTL7xAr5'9gӇY3؄K#K0?"ґO7ቦ$K_q%o/Oo%F.&ٗ lyX0s- 埘Mx%J^NC tJETH.{Zno(r% 8΅KnTB߾ 30'8$,%^\\HF"`wI";󖧿)+ _AyA$EH5 }SZ?2/HVdLm@vؠ$RtuS3\곑 u^$b(SI4wڅ"ŷ8 zA`{AYn=+"R%"5I'u܃G祎;@j<з_p)}^| $=^ \U-iɱ:;VLEN-x~^|L^̟x f p+""S%; scnhJӁxA@C @kǘy9KrfsB~l~[^՚,/HV&82͡k&}_:$a> vRmDdOٟ1@ y0i琝B߾d>0Co1@UUTΡP㞄nv'G()z)kKS1k`F8(OB>9xAa*"nL" 4ftiILrj^|YзVj^삧_}D˦_ |fT}w1D`#`5d; yXrYۇ?T,u=P6Œt -:ɷ,yn }oZ /H }ۑB~;8ug5Բ"^%z݊+ hRD4P%oqn:]74! `JdIQJ]T!is 25-"2(%"IbI~:nilZ#dMi`q~y{~ㅾ & |P-S0ID@l)k/" z81Ho` TV >Tr }0uSз>)VԤ=ҔTgߦZ:nUx >oз6 ztۗ6Hu(0X1_JŬnj;DC0P~3(ϒMtxM] l[U6з$U,>ȶlCi]'0洱sC~}0P[p:}V`சφ]з/VfdOx"ڱBߞ?oB7"جV7yOc{[rmOۯ}(p*ՎlڽRGdO!v'Clzh}Bp[ͿYjpHH5Vn:ղ㎳⨱o ɺ5OVƿ1%OobJYɏqIDATU.з.<[Sy}X~g63$oɦzja*v5wʴ=FA4p:M!ZŊb^^,٠z`u`>E1!_𽀙NvY5 Y,fsf߳IkfDy~lc,Ȼ 98HB~ {13^9&NQD@ ; оL'8zI`=X8/H.}i$ѡoM""#@ ;rSפ{llM!ҍ {ejH@j.@/a))˛ADD UcM ~"""2 (P oO5YTDDD*JEqKHnn:*(Tt@DDD JE*`EM!x DDDDP:h: Vi.TĊ{0SAexSATE HE 'Xq4 DDDDPBV= tRӭ8 DDDDPzg74M """R5%"h.pwqHKZO 7>t,R\Φ􊈈 'PzݛBDDDjJE5T+j +JE*% k:JKDDDdP(RIRM!YDDDDPZJ>EDDdXQ(R@*jTI H5TjlTI Hf7TjVTI Hm:~"""2(SM zDDDDPZ6T枦@ Yq:ϦJno:*)޹M GSBDDDJJEw6:t""""Ui:(uJqHikZq6H $`4 20%9M$/Ha~.1[<DDddR(RqBw8 o \r?fz-i5(/H&V$}p˘drз5=XDD@?&[qԵ\ YSۀBߞ5@_[וl^UxA,&ިf`sB~DDDB HMR ~X$'XqZӁMa8X¦~ :ȗ$_G$^,|8[qS8>-""R9%"5Jw!`c!lj(/HF.p4|] |# -ऊگ4d ``OSBߞQs_""" H8 l\t,2ǁ-4[n,M\/Hd~LR\w$pp$+?R=Z>++H$A!>Ё$.I# ^nT^JJ KTJB!7$K3 P^ aQ=:̝sNW>ZZf{~y'9xcg7HUQRl uK %V#^{NQR.!J?mamIPizqڎE<|85ȳk}%~{\u4Jxc]c8؂B4BA] lKYY~|<ȳS&4p:&Q_7/QRx$PsDJ# ΠA}!ȳ[havc0X8J$I˙J- cq,0wyDA +J-)[.x|QRx''FI񶶃$ `կy>W\ \Mٴn৔ VRt 24Q6XCˁM(?݁u\#Ύb4$Pj׽mL0ļ;NY,wlP7tVy‘$iV&σ<-0iʝt?lGyp;`Sw),<2zp1piggy}ANPc{[<:Tk<\:M1l4̯wJ|.Jqxj]wώ54ՒDIp@.'Q.IR+Q*i <9н<;xtw<<Qg]n?Jsi>?ϵNZ|RgR#8W() dTߙN8-I|xTuRV;o7-> TM>|'J`_פ0?q(J)!VMaI%IG@4gbtw ƣI݂ğ8I()N8u;()|,ß6!J(H{]QRRqv5QR \ጽG`_t)14ǽ"$iʙJMg߁}{E'<ٵ5=w)E}^'(vEIeўR^ؠz5eOIr!*Jˀ(?S~v3k)[EH*Jc_ %}~S9`YykQA;Pރ:}J3v $JcBeA+=)Lo=ü(K CZo ؖ\MOCst?LNuM JmoS>MF@()+[ Mm"IxJdonu_o~Gi5}X4ob?_d5i~7JӀO_O4N*Mu) `|H_ M4Ϧ85;uH_݁ =8<@$IZ $M4ϐ.nqUJ[U>oo1Ei~$IZ%@IPi> |}GY}s,w 8 x+]Q;1[X[9JK4}K^l#Xo(i>>.wdv4Op֓$i(&483Q /GJש]WR }/e-%x$I $M4أ]. ]KYmIa2()ΥlwC>y i.%PIFPR]lt'%znD7^Qʯ[8 ()> lC o ?Rz>gs9`Ipɟ$iZJz!ȳ  ϖyv#e*}t_7z>G$IPI UYFxKzgˀw$IZ f{3ȳ;;g \OYDrLN@@$@I[-GI$IՙJ \g 2I'ZA/ c-"IҘ04.^GYU?>X$Hai$_:[/~u>X$wnc/vnl5z<>Q=G5$I@irي.uNX4ȳfz3ȳeN[GXعw%IJ.?<{8rmRcnn< 䃔>u_SHjLtVgOfgO0=[,Ju+L97$IPL7 0fQQh!: ذ D$Ug(M5dj7ѢxMakSq $IcPҤ{1OzEIp1p[% V`a$I ]\仫*3M3M( |9J*U:ZU:_W $4Οg J"JӁ#8|dGI1eB!<̺$9&&Vg:__bt7lDs[AM4{QR p4p(CTqpt5$Ij 4\ \%!f&P\8Y KJ=s ^e2\xa?8xgM4OێE$*F{\zz.s {@I$-&F+lw 5ԕmt_2pd$ISPHym(ʆ+[|xGY8Ig(i<yp_1GC$ibJS,ȳ7$pcg{0ȳxpaytH$MKS.ȳ[z:ipp -a,$IcPZ <[ vu ^{0`z80ȳED&I4<*iyv9l H$5@I/ȳm;Iq$I$-&$I@J$Ia(I$I $I$-&\Mc$I"@Icnm< I$͋ A\ ,e#E$IC24 n>7ː/yvè$IpL% $ȳO ~Pgq+AI$It_t7m;I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I҈?*dFրIENDB`certstream-server-go-1.7.0/go.mod000066400000000000000000000014701473673250000167160ustar00rootroot00000000000000module github.com/d-Rickyy-b/certstream-server-go go 1.21.0 toolchain go1.22.3 require ( github.com/VictoriaMetrics/metrics v1.35.1 github.com/go-chi/chi/v5 v5.1.0 github.com/google/certificate-transparency-go v1.2.1 github.com/gorilla/websocket v1.5.3 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/go-logr/logr v1.4.2 // indirect github.com/google/trillian v1.6.0 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sys v0.23.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 // indirect google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect ) certstream-server-go-1.7.0/go.sum000066400000000000000000000071151473673250000167450ustar00rootroot00000000000000github.com/VictoriaMetrics/metrics v1.35.1 h1:o84wtBKQbzLdDy14XeskkCZih6anG+veZ1SwJHFGwrU= github.com/VictoriaMetrics/metrics v1.35.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/google/certificate-transparency-go v1.2.1 h1:4iW/NwzqOqYEEoCBEFP+jPbBXbLqMpq3CifMyOnDUME= github.com/google/certificate-transparency-go v1.2.1/go.mod h1:bvn/ytAccv+I6+DGkqpvSsEdiVGramgaSC6RD3tEmeE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/trillian v1.6.0 h1:jMBeDBIkINFvS2n6oV5maDqfRlxREAc6CW9QYWQ0qT4= github.com/google/trillian v1.6.0/go.mod h1:Yu3nIMITzNhhMJEHjAtp6xKiu+H/iHu2Oq5FjV2mCWI= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= certstream-server-go-1.7.0/internal/000077500000000000000000000000001473673250000174225ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/certificatetransparency/000077500000000000000000000000001473673250000243365ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/certificatetransparency/ct-parser.go000066400000000000000000000255051473673250000265740ustar00rootroot00000000000000package certificatetransparency import ( "bytes" "crypto/sha1" //nolint:gosec "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "fmt" "hash" "log" "math/big" "strings" "time" "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" ct "github.com/google/certificate-transparency-go" "github.com/google/certificate-transparency-go/x509" "github.com/google/certificate-transparency-go/x509/pkix" ) // parseData converts a *ct.RawLogEntry struct into a certstream.Data struct by copying some values and calculating others. func parseData(entry *ct.RawLogEntry, operatorName, logName, ctURL string) (certstream.Data, error) { certLink := fmt.Sprintf("%s/ct/v1/get-entries?start=%d&end=%d", ctURL, entry.Index, entry.Index) // Create main data structure data := certstream.Data{ CertIndex: entry.Index, CertLink: certLink, Seen: float64(time.Now().UnixMilli()) / 1_000, Source: certstream.Source{ Name: logName, URL: ctURL, Operator: operatorName, NormalizedURL: normalizeCtlogURL(ctURL), }, UpdateType: "X509LogEntry", } // Convert RawLogEntry to ct.LogEntry logEntry, conversionErr := entry.ToLogEntry() if conversionErr != nil { log.Println("Could not convert entry to LogEntry: ", conversionErr) return certstream.Data{}, conversionErr } var cert *x509.Certificate var rawData []byte var isPrecert bool switch { case logEntry.X509Cert != nil: cert = logEntry.X509Cert rawData = logEntry.X509Cert.Raw isPrecert = false case logEntry.Precert != nil: cert = logEntry.Precert.TBSCertificate rawData = logEntry.Precert.Submitted.Data isPrecert = true default: return certstream.Data{}, errors.New("could not parse entry: no certificate found") } // Calculate certificate hash from the raw DER bytes of the certificate data.LeafCert = leafCertFromX509cert(*cert) // recalculate hashes if the certificate is a precertificate if isPrecert { calculatedHash := calculateSHA1(rawData) data.LeafCert.Fingerprint = calculatedHash data.LeafCert.SHA1 = calculatedHash data.LeafCert.SHA256 = calculateSHA256(rawData) } certAsDER := base64.StdEncoding.EncodeToString(entry.Cert.Data) data.LeafCert.AsDER = certAsDER var parseErr error data.Chain, parseErr = parseCertificateChain(logEntry) if parseErr != nil { log.Println("Could not parse certificate chain: ", parseErr) return certstream.Data{}, parseErr } return data, nil } // parseCertificateChain returns the certificate chain in form of a []LeafCert from the given *ct.LogEntry. func parseCertificateChain(logEntry *ct.LogEntry) ([]certstream.LeafCert, error) { chain := make([]certstream.LeafCert, len(logEntry.Chain)) for i, chainEntry := range logEntry.Chain { myCert, parseErr := x509.ParseCertificate(chainEntry.Data) if parseErr != nil { log.Println("Error parsing certificate: ", parseErr) return nil, parseErr } leafCert := leafCertFromX509cert(*myCert) chain[i] = leafCert } return chain, nil } // leafCertFromX509cert converts a x509.Certificate to the custom LeafCert data structure. func leafCertFromX509cert(cert x509.Certificate) certstream.LeafCert { leafCert := certstream.LeafCert{ AllDomains: cert.DNSNames, Extensions: certstream.Extensions{}, NotAfter: cert.NotAfter.Unix(), NotBefore: cert.NotBefore.Unix(), SerialNumber: formatSerialNumber(cert.SerialNumber), SignatureAlgorithm: parseSignatureAlgorithm(cert.SignatureAlgorithm), IsCA: cert.IsCA, } // The zero value of DomainsEntry.Data is nil, but we want an empty array - especially for json marshalling later. if leafCert.AllDomains == nil { leafCert.AllDomains = []string{} } leafCert.Subject = buildSubject(cert.Subject) if *leafCert.Subject.CN != "" && !leafCert.IsCA { domainAlreadyAdded := false // TODO check if CN matches domain regex for _, domain := range leafCert.AllDomains { if domain == *leafCert.Subject.CN { domainAlreadyAdded = true break } } if !domainAlreadyAdded { leafCert.AllDomains = append(leafCert.AllDomains, *leafCert.Subject.CN) } } leafCert.Issuer = buildSubject(cert.Issuer) leafCert.AsDER = base64.StdEncoding.EncodeToString(cert.Raw) leafCert.Fingerprint = calculateSHA1(cert.Raw) leafCert.SHA1 = leafCert.Fingerprint leafCert.SHA256 = calculateSHA256(cert.Raw) // TODO fix Extensions - check x509util.go for _, extension := range cert.Extensions { switch { case extension.Id.Equal(x509.OIDExtensionAuthorityKeyId): leafCert.Extensions.AuthorityKeyIdentifier = formatKeyID(cert.AuthorityKeyId) case extension.Id.Equal(x509.OIDExtensionKeyUsage): keyUsage := keyUsageToString(cert.KeyUsage) leafCert.Extensions.KeyUsage = &keyUsage case extension.Id.Equal(x509.OIDExtensionSubjectKeyId): leafCert.Extensions.SubjectKeyIdentifier = formatKeyID(cert.SubjectKeyId) case extension.Id.Equal(x509.OIDExtensionBasicConstraints): isCA := strings.ToUpper(fmt.Sprintf("CA:%t", cert.IsCA)) leafCert.Extensions.BasicConstraints = &isCA case extension.Id.Equal(x509.OIDExtensionSubjectAltName): var buf bytes.Buffer for _, name := range cert.DNSNames { commaAppend(&buf, "DNS:"+name) } for _, email := range cert.EmailAddresses { commaAppend(&buf, "email:"+email) } for _, ip := range cert.IPAddresses { commaAppend(&buf, "IP Address:"+ip.String()) } subjectAltName := buf.String() leafCert.Extensions.SubjectAltName = &subjectAltName case extension.Id.Equal(x509.OIDExtensionAuthorityInfoAccess): var buf bytes.Buffer for _, issuer := range cert.IssuingCertificateURL { commaAppend(&buf, "URI:"+issuer) } for _, ocsp := range cert.OCSPServer { commaAppend(&buf, "URI:"+ocsp) } result := buf.String() leafCert.Extensions.AuthorityInfoAccess = &result case extension.Id.Equal(x509.OIDExtensionCTPoison): leafCert.Extensions.CTLPoisonByte = true } } return leafCert } // buildSubject generates a Subject struct from the given pkix.Name. func buildSubject(certSubject pkix.Name) certstream.Subject { subject := certstream.Subject{ C: parseName(certSubject.Country), CN: &certSubject.CommonName, L: parseName(certSubject.Locality), O: parseName(certSubject.Organization), OU: parseName(certSubject.OrganizationalUnit), ST: parseName(certSubject.StreetAddress), } var aggregated string if subject.C != nil { aggregated += fmt.Sprintf("/C=%s", *subject.C) } if subject.CN != nil { aggregated += fmt.Sprintf("/CN=%s", *subject.CN) } if subject.L != nil { aggregated += fmt.Sprintf("/L=%s", *subject.L) } if subject.O != nil { aggregated += fmt.Sprintf("/O=%s", *subject.O) } if subject.OU != nil { aggregated += fmt.Sprintf("/OU=%s", *subject.OU) } if subject.ST != nil { aggregated += fmt.Sprintf("/ST=%s", *subject.ST) } subject.Aggregated = &aggregated return subject } // formatKeyID transforms the AuthorityKeyIdentifier to be more readable. func formatKeyID(keyID []byte) *string { tmp := hex.EncodeToString(keyID) var digest string for i := 0; i < len(tmp); i += 2 { digest = digest + ":" + tmp[i:i+2] } digest = strings.TrimLeft(digest, ":") digest = fmt.Sprintf("keyid:%s", digest) return &digest } func formatSerialNumber(serialNumber *big.Int) string { sn := fmt.Sprintf("%X", serialNumber) if len(sn)%2 == 1 { sn = "0" + sn } return sn } func parseName(input []string) *string { if input == nil { return nil } var result string for _, s := range input { if len(result) > 0 { result += "," } result += s } return &result } // calculateHash takes a hash.Hash struct and calculates the fingerprint of the given data. func calculateHash(data []byte, certHasher hash.Hash) string { _, e := certHasher.Write(data) if e != nil { log.Printf("Error while hashing cert: %s\n", e) return "" } certHash := fmt.Sprintf("%02x", certHasher.Sum(nil)) certHash = strings.ToUpper(certHash) var result bytes.Buffer for i := 0; i < len(certHash); i++ { if i%2 == 0 && i > 0 { result.WriteByte(':') } c := certHash[i] result.WriteByte(c) } return result.String() } // calculateSHA1 calculates the SHA1 fingerprint of the given data. func calculateSHA1(data []byte) string { return calculateHash(data, sha1.New()) //nolint:gosec } // calculateSHA256 calculates the SHA256 fingerprint of the given data. func calculateSHA256(data []byte) string { return calculateHash(data, sha256.New()) } func parseSignatureAlgorithm(signatureAlgoritm x509.SignatureAlgorithm) string { switch signatureAlgoritm { case x509.MD2WithRSA: return "md2, rsa" case x509.MD5WithRSA: return "md5, rsa" case x509.SHA1WithRSA: return "sha1, rsa" case x509.SHA256WithRSA: return "sha256, rsa" case x509.SHA384WithRSA: return "sha384, rsa" case x509.SHA512WithRSA: return "sha512, rsa" case x509.SHA256WithRSAPSS: return "sha256, rsa-pss" case x509.SHA384WithRSAPSS: return "sha384, rsa-pss" case x509.SHA512WithRSAPSS: return "sha512, rsa-pss" case x509.DSAWithSHA1: return "dsa, sha1" case x509.DSAWithSHA256: return "dsa, sha256" case x509.ECDSAWithSHA1: return "ecdsa, sha1" case x509.ECDSAWithSHA256: return "ecdsa, sha256" case x509.ECDSAWithSHA384: return "ecdsa, sha384" case x509.ECDSAWithSHA512: return "ecdsa, sha512" case x509.PureEd25519: return "ed25519" case x509.UnknownSignatureAlgorithm: fallthrough default: return "unknown" } } // commaAppend lets you append a string with a comma prepended to a buffer. func commaAppend(buf *bytes.Buffer, s string) { if buf.Len() > 0 { buf.WriteString(", ") } buf.WriteString(s) } func keyUsageToString(k x509.KeyUsage) string { var buf bytes.Buffer if k&x509.KeyUsageDigitalSignature != 0 { commaAppend(&buf, "Digital Signature") } if k&x509.KeyUsageContentCommitment != 0 { commaAppend(&buf, "Content Commitment") } if k&x509.KeyUsageKeyEncipherment != 0 { commaAppend(&buf, "Key Encipherment") } if k&x509.KeyUsageDataEncipherment != 0 { commaAppend(&buf, "Data Encipherment") } if k&x509.KeyUsageKeyAgreement != 0 { commaAppend(&buf, "Key Agreement") } if k&x509.KeyUsageCertSign != 0 { commaAppend(&buf, "Certificate Signing") } if k&x509.KeyUsageCRLSign != 0 { commaAppend(&buf, "CRL Signing") } if k&x509.KeyUsageEncipherOnly != 0 { commaAppend(&buf, "Encipher Only") } if k&x509.KeyUsageDecipherOnly != 0 { commaAppend(&buf, "Decipher Only") } return buf.String() } // parseCertstreamEntry creates an Entry from a ct.RawLogEntry. func parseCertstreamEntry(rawEntry *ct.RawLogEntry, operatorName, logname, ctURL string) (certstream.Entry, error) { if rawEntry == nil { return certstream.Entry{}, errors.New("certstream entry is nil") } data, err := parseData(rawEntry, operatorName, logname, ctURL) if err != nil { return certstream.Entry{}, err } entry := certstream.Entry{ Data: data, MessageType: "certificate_update", } return entry, nil } certstream-server-go-1.7.0/internal/certificatetransparency/ct-watcher.go000066400000000000000000000226631473673250000267370ustar00rootroot00000000000000package certificatetransparency import ( "context" "errors" "fmt" "io" "log" "net/http" "strings" "sync" "sync/atomic" "time" "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/d-Rickyy-b/certstream-server-go/internal/web" ct "github.com/google/certificate-transparency-go" "github.com/google/certificate-transparency-go/client" "github.com/google/certificate-transparency-go/jsonclient" "github.com/google/certificate-transparency-go/loglist3" "github.com/google/certificate-transparency-go/scanner" ) var ( errCreatingClient = errors.New("failed to create JSON client") errFetchingSTHFailed = errors.New("failed to fetch STH") userAgent = fmt.Sprintf("Certstream Server v%s (github.com/d-Rickyy-b/certstream-server-go)", config.Version) ) // Watcher describes a component that watches for new certificates in a CT log. type Watcher struct { workers []*worker wg sync.WaitGroup context context.Context certChan chan certstream.Entry cancelFunc context.CancelFunc } // NewWatcher creates a new Watcher. func NewWatcher(certChan chan certstream.Entry) *Watcher { return &Watcher{ certChan: certChan, } } // Start starts the watcher. This method is blocking. func (w *Watcher) Start() { w.context, w.cancelFunc = context.WithCancel(context.Background()) // Create new certChan if it doesn't exist yet if w.certChan == nil { w.certChan = make(chan certstream.Entry, 5000) } // initialize the watcher with currently available logs w.addNewlyAvailableLogs() log.Println("Started CT watcher") go certHandler(w.certChan) go w.watchNewLogs() w.wg.Wait() close(w.certChan) } // watchNewLogs monitors the ct log list for new logs and starts a worker for each new log found. // This method is blocking. It can be stopped by cancelling the context. func (w *Watcher) watchNewLogs() { // Add all available logs to the watcher w.addNewlyAvailableLogs() // Check for new logs once every hour ticker := time.NewTicker(1 * time.Hour) for { select { case <-ticker.C: w.addNewlyAvailableLogs() case <-w.context.Done(): ticker.Stop() return } } } // The transparency log list is constantly updated with new Log servers. // This function checks for new ct logs and adds them to the watcher. func (w *Watcher) addNewlyAvailableLogs() { log.Println("Checking for new ct logs...") // Get a list of urls of all CT logs logList, err := getAllLogs() if err != nil { log.Println(err) return } newCTs := 0 // Check the ct log list for new, unwatched logs // For each CT log, create a worker and start downloading certs for _, operator := range logList.Operators { // Iterate over each log of the operator for _, transparencyLog := range operator.Logs { // Check if the log is already being watched newURL := normalizeCtlogURL(transparencyLog.URL) alreadyWatched := false for _, ctWorker := range w.workers { workerURL := normalizeCtlogURL(ctWorker.ctURL) if workerURL == newURL { alreadyWatched = true break } } // TODO maybe add a check for logs that are still watched but no longer on the logList and remove them? See also issue #41 and #42 // If the log is not being watched, create a new worker if !alreadyWatched { w.wg.Add(1) newCTs++ ctWorker := worker{ name: transparencyLog.Description, operatorName: operator.Name, ctURL: transparencyLog.URL, entryChan: w.certChan, } w.workers = append(w.workers, &ctWorker) // Start a goroutine for each worker go func() { defer w.wg.Done() ctWorker.startDownloadingCerts(w.context) }() } } } log.Printf("New ct logs found: %d\n", newCTs) log.Printf("Currently monitored ct logs: %d\n", len(w.workers)) } // Stop stops the watcher. func (w *Watcher) Stop() { log.Printf("Stopping watcher\n") w.cancelFunc() } // A worker processes a single CT log. type worker struct { name string operatorName string ctURL string entryChan chan certstream.Entry mu sync.Mutex running bool } // startDownloadingCerts starts downloading certificates from the CT log. This method is blocking. func (w *worker) startDownloadingCerts(ctx context.Context) { // Normalize CT URL. We remove trailing slashes and prepend "https://" if it's not already there. w.ctURL = strings.TrimRight(w.ctURL, "/") if !strings.HasPrefix(w.ctURL, "https://") && !strings.HasPrefix(w.ctURL, "http://") { w.ctURL = "https://" + w.ctURL } log.Printf("Starting worker for CT log: %s\n", w.ctURL) defer log.Printf("Stopping worker for CT log: %s\n", w.ctURL) w.mu.Lock() if w.running { log.Printf("Worker for '%s' already running\n", w.ctURL) w.mu.Unlock() return } w.running = true w.mu.Unlock() for { workerErr := w.runWorker(ctx) if workerErr != nil { if errors.Is(workerErr, errFetchingSTHFailed) { log.Printf("Worker for '%s' failed - could not fetch STH\n", w.ctURL) return } else if errors.Is(workerErr, errCreatingClient) { log.Printf("Worker for '%s' failed - could not create client\n", w.ctURL) return } else if strings.Contains(workerErr.Error(), "no such host") { log.Printf("Worker for '%s' failed to resolve host: %s\n", w.ctURL, workerErr) return } log.Printf("Worker for '%s' failed with unexpected error: %s\n", w.ctURL, workerErr) } // Check if the context was cancelled select { case <-ctx.Done(): log.Printf("Context was cancelled; Stopping worker for '%s'\n", w.ctURL) return default: log.Printf("Worker for '%s' sleeping for 5 seconds due to error\n", w.ctURL) time.Sleep(5 * time.Second) log.Printf("Restarting worker for '%s'\n", w.ctURL) continue } } } // runWorker runs a single worker for a single CT log. This method is blocking. func (w *worker) runWorker(ctx context.Context) error { hc := http.Client{Timeout: 30 * time.Second} jsonClient, e := client.New(w.ctURL, &hc, jsonclient.Options{UserAgent: userAgent}) if e != nil { log.Printf("Error creating JSON client: %s\n", e) return errCreatingClient } sth, getSTHerr := jsonClient.GetSTH(ctx) if getSTHerr != nil { log.Printf("Could not get STH for '%s': %s\n", w.ctURL, getSTHerr) return errFetchingSTHFailed } certScanner := scanner.NewScanner(jsonClient, scanner.ScannerOptions{ FetcherOptions: scanner.FetcherOptions{ BatchSize: 100, ParallelFetch: 1, StartIndex: int64(sth.TreeSize), // Start at the latest STH to skip all the past certificates Continuous: true, }, Matcher: scanner.MatchAll{}, PrecertOnly: false, NumWorkers: 1, BufferSize: 1000, }) scanErr := certScanner.Scan(ctx, w.foundCertCallback, w.foundPrecertCallback) if scanErr != nil { log.Println("Scan error: ", scanErr) return scanErr } log.Println("No error from certScanner!") return nil } // foundCertCallback is the callback that handles cases where new regular certs are found. func (w *worker) foundCertCallback(rawEntry *ct.RawLogEntry) { entry, parseErr := parseCertstreamEntry(rawEntry, w.operatorName, w.name, w.ctURL) if parseErr != nil { log.Println("Error parsing certstream entry: ", parseErr) return } entry.Data.UpdateType = "X509LogEntry" w.entryChan <- entry atomic.AddInt64(&processedCerts, 1) } // foundPrecertCallback is the callback that handles cases where new precerts are found. func (w *worker) foundPrecertCallback(rawEntry *ct.RawLogEntry) { entry, parseErr := parseCertstreamEntry(rawEntry, w.operatorName, w.name, w.ctURL) if parseErr != nil { log.Println("Error parsing certstream entry: ", parseErr) return } entry.Data.UpdateType = "PrecertLogEntry" w.entryChan <- entry atomic.AddInt64(&processedPrecerts, 1) } // certHandler takes the entries out of the entryChan channel and broadcasts them to all clients. // Only a single instance of the certHandler runs per certstream server. func certHandler(entryChan chan certstream.Entry) { var processed int64 for { entry := <-entryChan processed++ if processed%1000 == 0 { log.Printf("Processed %d entries | Queue length: %d\n", processed, len(entryChan)) // Every thousandth entry, we store one certificate as example web.SetExampleCert(entry) } // Run json encoding in the background and send the result to the clients. web.ClientHandler.Broadcast <- entry // Update metrics url := entry.Data.Source.NormalizedURL operator := entry.Data.Source.Operator metrics.Inc(operator, url) } } // getAllLogs returns a list of all CT logs. func getAllLogs() (loglist3.LogList, error) { // Download the list of all logs from ctLogInfo and decode json resp, err := http.Get(loglist3.LogListURL) if err != nil { return loglist3.LogList{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return loglist3.LogList{}, errors.New("failed to download loglist") } bodyBytes, readErr := io.ReadAll(resp.Body) if readErr != nil { log.Panic(readErr) } allLogs, parseErr := loglist3.NewFromJSON(bodyBytes) if parseErr != nil { return loglist3.LogList{}, parseErr } // Add new ct logs to metrics for _, operator := range allLogs.Operators { for _, ctlog := range operator.Logs { url := normalizeCtlogURL(ctlog.URL) metrics.Init(operator.Name, url) } } return *allLogs, nil } func normalizeCtlogURL(input string) string { input = strings.TrimPrefix(input, "https://") input = strings.TrimPrefix(input, "http://") input = strings.TrimSuffix(input, "/") return input } certstream-server-go-1.7.0/internal/certificatetransparency/logmetrics.go000066400000000000000000000064771473673250000270530ustar00rootroot00000000000000package certificatetransparency import "sync" type ( // OperatorLogs is a map of operator names to a list of CT log urls, operated by said operator. OperatorLogs map[string][]string // OperatorMetric is a map of CT log urls to the number of certs processed by said log. OperatorMetric map[string]int64 // CTMetrics is a map of operator names to a map of CT log urls to the number of certs processed by said log. CTMetrics map[string]OperatorMetric ) var ( processedCerts int64 processedPrecerts int64 metrics = LogMetrics{metrics: make(CTMetrics)} ) // LogMetrics is a struct that holds a map of metrics for each CT log grouped by operator. // Metrics can be accessed and written concurrently through the Get, Set and Inc methods. type LogMetrics struct { mutex sync.RWMutex metrics CTMetrics } // GetCTMetrics returns a copy of the internal metrics map. func (m *LogMetrics) GetCTMetrics() CTMetrics { m.mutex.RLock() defer m.mutex.RUnlock() copiedMap := make(CTMetrics) for operator, urls := range m.metrics { copiedMap[operator] = make(OperatorMetric) for url, count := range urls { copiedMap[operator][url] = count } } return copiedMap } // OperatorLogMapping returns a map of operator names to a list of CT logs. func (m *LogMetrics) OperatorLogMapping() OperatorLogs { m.mutex.RLock() defer m.mutex.RUnlock() logOperators := make(map[string][]string, len(m.metrics)) for operator, urls := range m.metrics { urlList := make([]string, len(urls)) counter := 0 for url := range urls { urlList[counter] = url counter++ } logOperators[operator] = urlList } return logOperators } // Init initializes the internal metrics map with the given operator names and CT log urls if it doesn't exist yet. func (m *LogMetrics) Init(operator, url string) { m.mutex.Lock() defer m.mutex.Unlock() // if the operator does not exist, create a new entry if _, ok := m.metrics[operator]; !ok { m.metrics[operator] = make(OperatorMetric) } // if the operator exists but the url does not, create a new entry if _, ok := m.metrics[operator][url]; !ok { m.metrics[operator][url] = 0 } } // Get the metric for a given operator and ct url. func (m *LogMetrics) Get(operator, url string) int64 { // Despite this being a getter, we still need to fully lock the mutex because we might modify the map if the requested operator does not exist. m.mutex.Lock() defer m.mutex.Unlock() if _, ok := m.metrics[operator]; !ok { m.metrics[operator] = make(OperatorMetric) } return m.metrics[operator][url] } // Set the metric for a given operator and ct url. func (m *LogMetrics) Set(operator, url string, value int64) { m.mutex.Lock() defer m.mutex.Unlock() if _, ok := m.metrics[operator]; !ok { m.metrics[operator] = make(OperatorMetric) } m.metrics[operator][url] = value } // Inc the metric for a given operator and ct url. func (m *LogMetrics) Inc(operator, url string) { m.mutex.Lock() defer m.mutex.Unlock() if _, ok := m.metrics[operator]; !ok { m.metrics[operator] = make(OperatorMetric) } m.metrics[operator][url]++ } func GetProcessedCerts() int64 { return processedCerts } func GetProcessedPrecerts() int64 { return processedPrecerts } func GetCertMetrics() CTMetrics { return metrics.GetCTMetrics() } func GetLogOperators() map[string][]string { return metrics.OperatorLogMapping() } certstream-server-go-1.7.0/internal/certstream/000077500000000000000000000000001473673250000215735ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/certstream/models.go000066400000000000000000000103671473673250000234140ustar00rootroot00000000000000package certstream import ( "bytes" "encoding/json" "log" ) type Entry struct { Data Data `json:"data"` MessageType string `json:"message_type"` cachedJSON []byte cachedJSONLite []byte } // Clone returns a new copy of the Entry. func (e *Entry) Clone() Entry { return Entry{ Data: e.Data, MessageType: e.MessageType, cachedJSON: e.cachedJSON, cachedJSONLite: e.cachedJSONLite, } } // JSON returns the json encoded Entry as byte slice and caches it for later access. func (e *Entry) JSON() []byte { if len(e.cachedJSON) > 0 { return e.cachedJSON } e.cachedJSON = e.entryToJSONBytes() return e.cachedJSON } // JSONNoCache returns the json encoded Entry as byte slice without caching it. func (e *Entry) JSONNoCache() []byte { return e.entryToJSONBytes() } // JSONLite does the same as JSON() but removes the chain and cert's DER representation. func (e *Entry) JSONLite() []byte { if len(e.cachedJSONLite) > 0 { return e.cachedJSONLite } e.cachedJSONLite = e.JSONLiteNoCache() return e.cachedJSONLite } // JSONLiteNoCache does the same as JSONNoCache() but removes the chain and cert's DER representation. func (e *Entry) JSONLiteNoCache() []byte { newEntry := e.Clone() newEntry.Data.Chain = nil newEntry.Data.LeafCert.AsDER = "" return newEntry.entryToJSONBytes() } // JSONDomains returns the json encoded domains (DomainsEntry) as byte slice. func (e *Entry) JSONDomains() []byte { domainsEntry := DomainsEntry{ Data: e.Data.LeafCert.AllDomains, MessageType: "dns_entries", } domainsEntryBytes, err := json.Marshal(domainsEntry) if err != nil { log.Println(err) } return domainsEntryBytes } // entryToJSONBytes encodes an Entry to a JSON byte slice. func (e *Entry) entryToJSONBytes() []byte { buf := bytes.Buffer{} enc := json.NewEncoder(&buf) enc.SetEscapeHTML(false) err := enc.Encode(e) if err != nil { log.Println(err) } return buf.Bytes() } type Data struct { CertIndex int64 `json:"cert_index"` CertLink string `json:"cert_link"` Chain []LeafCert `json:"chain,omitempty"` LeafCert LeafCert `json:"leaf_cert"` Seen float64 `json:"seen"` Source Source `json:"source"` UpdateType string `json:"update_type"` } type Source struct { Name string `json:"name"` URL string `json:"url"` Operator string `json:"-"` NormalizedURL string `json:"-"` } type LeafCert struct { AllDomains []string `json:"all_domains"` AsDER string `json:"as_der,omitempty"` Extensions Extensions `json:"extensions"` Fingerprint string `json:"fingerprint"` SHA1 string `json:"sha1"` SHA256 string `json:"sha256"` NotAfter int64 `json:"not_after"` NotBefore int64 `json:"not_before"` SerialNumber string `json:"serial_number"` SignatureAlgorithm string `json:"signature_algorithm"` Subject Subject `json:"subject"` Issuer Subject `json:"issuer"` IsCA bool `json:"is_ca"` } type Subject struct { C *string `json:"C"` CN *string `json:"CN"` L *string `json:"L"` O *string `json:"O"` OU *string `json:"OU"` ST *string `json:"ST"` Aggregated *string `json:"aggregated"` EmailAddress *string `json:"email_address"` } type Extensions struct { AuthorityInfoAccess *string `json:"authorityInfoAccess,omitempty"` AuthorityKeyIdentifier *string `json:"authorityKeyIdentifier,omitempty"` BasicConstraints *string `json:"basicConstraints,omitempty"` CertificatePolicies *string `json:"certificatePolicies,omitempty"` CtlSignedCertificateTimestamp *string `json:"ctlSignedCertificateTimestamp,omitempty"` ExtendedKeyUsage *string `json:"extendedKeyUsage,omitempty"` KeyUsage *string `json:"keyUsage,omitempty"` SubjectAltName *string `json:"subjectAltName,omitempty"` SubjectKeyIdentifier *string `json:"subjectKeyIdentifier,omitempty"` CTLPoisonByte bool `json:"ctlPoisonByte,omitempty"` } type DomainsEntry struct { Data []string `json:"data"` MessageType string `json:"message_type"` } certstream-server-go-1.7.0/internal/config/000077500000000000000000000000001473673250000206675ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/config/config.go000066400000000000000000000120771473673250000224720ustar00rootroot00000000000000package config import ( "log" "net" "os" "path/filepath" "regexp" "strings" "gopkg.in/yaml.v3" ) var ( AppConfig Config Version = "1.6.0" ) type ServerConfig struct { ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` CertPath string `yaml:"cert_path"` CertKeyPath string `yaml:"cert_key_path"` RealIP bool `yaml:"real_ip"` Whitelist []string `yaml:"whitelist"` } type Config struct { Webserver struct { ServerConfig `yaml:",inline"` FullURL string `yaml:"full_url"` LiteURL string `yaml:"lite_url"` DomainsOnlyURL string `yaml:"domains_only_url"` CompressionEnabled bool `yaml:"compression_enabled"` } Prometheus struct { ServerConfig `yaml:",inline"` Enabled bool `yaml:"enabled"` MetricsURL string `yaml:"metrics_url"` ExposeSystemMetrics bool `yaml:"expose_system_metrics"` } } // ReadConfig reads the config file and returns a filled Config struct. func ReadConfig(configPath string) (Config, error) { log.Printf("Reading config file '%s'...\n", configPath) conf, parseErr := parseConfigFromFile(configPath) if parseErr != nil { log.Fatalln("Error while parsing yaml file:", parseErr) } if !validateConfig(conf) { log.Fatalln("Invalid config") } AppConfig = conf return conf, nil } // parseConfigFromFile reads the config file as bytes and passes it to parseConfigFromBytes. // It returns a filled Config struct. func parseConfigFromFile(configFile string) (Config, error) { if configFile == "" { configFile = "config.yml" } // Check if the file exists absPath, err := filepath.Abs(configFile) if err != nil { log.Printf("Couldn't convert to absolute path: '%s'\n", configFile) return Config{}, err } if _, statErr := os.Stat(absPath); os.IsNotExist(statErr) { log.Printf("Config file '%s' does not exist\n", absPath) ext := filepath.Ext(absPath) absPath = strings.TrimSuffix(absPath, ext) switch ext { case ".yaml": absPath += ".yml" case ".yml": absPath += ".yaml" default: log.Printf("Config file '%s' does not have a valid extension\n", configFile) return Config{}, statErr } if _, secondStatErr := os.Stat(absPath); os.IsNotExist(secondStatErr) { log.Printf("Config file '%s' does not exist\n", absPath) return Config{}, secondStatErr } } log.Printf("File '%s' exists\n", absPath) yamlFileContent, readErr := os.ReadFile(absPath) if readErr != nil { return Config{}, readErr } conf, parseErr := parseConfigFromBytes(yamlFileContent) if parseErr != nil { return Config{}, parseErr } return conf, nil } // parseConfigFromBytes parses the config bytes and returns a filled Config struct. func parseConfigFromBytes(data []byte) (Config, error) { var config Config err := yaml.Unmarshal(data, &config) if err != nil { return config, err } return config, nil } // validateConfig validates the config values and sets defaults for missing values. func validateConfig(config Config) bool { // Still matches invalid IP addresses but good enough for detecting completely wrong formats URLRegex := regexp.MustCompile(`^(/[a-zA-Z0-9\-._]+)+$`) // Check webserver config if config.Webserver.ListenAddr == "" || net.ParseIP(config.Webserver.ListenAddr) == nil { log.Fatalln("Webhook listen IP is not a valid IP: ", config.Webserver.ListenAddr) return false } if config.Webserver.ListenPort == 0 { log.Fatalln("Webhook listen port is not set") return false } if config.Webserver.FullURL == "" || !URLRegex.MatchString(config.Webserver.FullURL) { log.Println("Webhook full URL is not set or does not match pattern '/...'") config.Webserver.FullURL = "/full-stream" } if config.Webserver.LiteURL == "" || !URLRegex.MatchString(config.Webserver.FullURL) { log.Println("Webhook lite URL is not set or does not match pattern '/...'") config.Webserver.LiteURL = "/" } if config.Webserver.DomainsOnlyURL == "" || !URLRegex.MatchString(config.Webserver.DomainsOnlyURL) { log.Println("Webhook domains only URL is not set or does not match pattern '/...'") config.Webserver.FullURL = "/domains-only" } if config.Webserver.FullURL == config.Webserver.LiteURL { log.Fatalln("Webhook full URL is the same as lite URL - please fix the config!") } if config.Webserver.DomainsOnlyURL == "" { config.Webserver.FullURL = "/domains-only" } if config.Prometheus.Enabled { if config.Prometheus.ListenAddr == "" || net.ParseIP(config.Prometheus.ListenAddr) == nil { log.Fatalln("Metrics export IP is not a valid IP") return false } if config.Prometheus.ListenPort == 0 { log.Fatalln("Metrics export port is not set") return false } if config.Prometheus.Whitelist == nil { config.Prometheus.Whitelist = []string{} } // Check if IPs in whitelist match pattern for _, ip := range config.Prometheus.Whitelist { if net.ParseIP(ip) == nil { // Provided entry is not an IP, check if it's a CIDR range _, _, err := net.ParseCIDR(ip) if err != nil { log.Fatalln("Invalid IP in metrics whitelist: ", ip) return false } } } } return true } certstream-server-go-1.7.0/internal/metrics/000077500000000000000000000000001473673250000210705ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/metrics/prometheus.go000066400000000000000000000100611473673250000236100ustar00rootroot00000000000000package metrics import ( "fmt" "io" "strings" "sync" "time" "github.com/d-Rickyy-b/certstream-server-go/internal/certificatetransparency" "github.com/d-Rickyy-b/certstream-server-go/internal/web" "github.com/VictoriaMetrics/metrics" ) var ( ctLogMetricsInitialized = false ctLogMetricsInitMutex = &sync.Mutex{} tempCertMetricsLastRefreshed = time.Time{} tempCertMetrics = certificatetransparency.CTMetrics{} // Number of currently connected clients. fullClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"full\"}", func() float64 { return float64(web.ClientHandler.ClientFullCount()) }) liteClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"lite\"}", func() float64 { return float64(web.ClientHandler.ClientLiteCount()) }) domainClientCount = metrics.NewGauge("certstreamservergo_clients_total{type=\"domain\"}", func() float64 { return float64(web.ClientHandler.ClientDomainsCount()) }) // Number of certificates processed by the CT watcher. processedCertificates = metrics.NewGauge("certstreamservergo_certificates_total{type=\"regular\"}", func() float64 { return float64(certificatetransparency.GetProcessedCerts()) }) processedPreCertificates = metrics.NewGauge("certstreamservergo_certificates_total{type=\"precert\"}", func() float64 { return float64(certificatetransparency.GetProcessedPrecerts()) }) ) // WritePrometheus provides an easy way to write metrics to a writer. func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { ctLogMetricsInitMutex.Lock() if !ctLogMetricsInitialized { initCtLogMetrics() } ctLogMetricsInitMutex.Unlock() getSkippedCertMetrics() metrics.WritePrometheus(w, exposeProcessMetrics) } // For having metrics regarding each individual CT log, we need to register them manually. // initCtLogMetrics fetches all the CT Logs and registers one metric per log. func initCtLogMetrics() { logs := certificatetransparency.GetLogOperators() for operator, urls := range logs { operator := operator // Copy variable to new scope for i := 0; i < len(urls); i++ { url := urls[i] name := fmt.Sprintf("certstreamservergo_certs_by_log_total{url=\"%s\",operator=\"%s\"}", url, operator) metrics.NewGauge(name, func() float64 { return float64(getCertCountForLog(operator, url)) }) } } if len(logs) > 0 { ctLogMetricsInitialized = true } } // getCertCountForLog returns the number of certificates processed from a specific CT log. // It caches the result for 5 seconds. Subsequent calls to this method will return the cached result. func getCertCountForLog(operatorName, logname string) int64 { // Add some caching to avoid having to lock the mutex every time if time.Since(tempCertMetricsLastRefreshed) > time.Second*5 { tempCertMetricsLastRefreshed = time.Now() tempCertMetrics = certificatetransparency.GetCertMetrics() } return tempCertMetrics[operatorName][logname] } // getSkippedCertMetrics gets the number of skipped certificates for each client and creates metrics for it. // It also removes metrics for clients that are not connected anymore. func getSkippedCertMetrics() { skippedCerts := web.ClientHandler.GetSkippedCerts() for clientName := range skippedCerts { // Get or register a new counter for each client metricName := fmt.Sprintf("certstreamservergo_skipped_certs{client=\"%s\"}", clientName) c := metrics.GetOrCreateCounter(metricName) c.Set(skippedCerts[clientName]) } // Remove all metrics that are not in the list of current client skipped cert metrics // Get a list of current client skipped cert metrics for _, metricName := range metrics.ListMetricNames() { if !strings.HasPrefix(metricName, "certstreamservergo_skipped_certs") { continue } clientName := strings.TrimPrefix(metricName, "certstreamservergo_skipped_certs{client=\"") clientName = strings.TrimSuffix(clientName, "\"}") // Check if the registered metric is in the list of current client skipped cert metrics // If not, unregister the metric _, exists := skippedCerts[clientName] if !exists { metrics.UnregisterMetric(metricName) } } } certstream-server-go-1.7.0/internal/web/000077500000000000000000000000001473673250000201775ustar00rootroot00000000000000certstream-server-go-1.7.0/internal/web/broadcastmanager.go000066400000000000000000000071141473673250000240260ustar00rootroot00000000000000package web import ( "log" "sync" "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" ) type BroadcastManager struct { Broadcast chan certstream.Entry clients []*client clientLock sync.RWMutex } // registerClient adds a client to the list of clients of the BroadcastManager. // The client will receive certificate broadcasts right after registration. func (bm *BroadcastManager) registerClient(c *client) { bm.clientLock.Lock() bm.clients = append(bm.clients, c) log.Printf("Clients: %d, Capacity: %d\n", len(bm.clients), cap(bm.clients)) bm.clientLock.Unlock() } // unregisterClient removes a client from the list of clients of the BroadcastManager. // The client will no longer receive certificate broadcasts right after unregistering. func (bm *BroadcastManager) unregisterClient(c *client) { bm.clientLock.Lock() for i, client := range bm.clients { if c == client { // Copy the last element of the slice to the position of the removed element // Then remove the last element by re-slicing bm.clients[i] = bm.clients[len(bm.clients)-1] bm.clients[len(bm.clients)-1] = nil bm.clients = bm.clients[:len(bm.clients)-1] // Close the broadcast channel of the client, otherwise this leads to a memory leak close(c.broadcastChan) break } } bm.clientLock.Unlock() } // ClientFullCount returns the current number of clients connected to the service on the `full` endpoint. func (bm *BroadcastManager) ClientFullCount() (count int64) { return bm.clientCountByType(SubTypeFull) } // ClientLiteCount returns the current number of clients connected to the service on the `lite` endpoint. func (bm *BroadcastManager) ClientLiteCount() (count int64) { return bm.clientCountByType(SubTypeLite) } // ClientDomainsCount returns the current number of clients connected to the service on the `domains-only` endpoint. func (bm *BroadcastManager) ClientDomainsCount() (count int64) { return bm.clientCountByType(SubTypeDomain) } // clientCountByType returns the current number of clients connected to the service on the endpoint matching // the specified SubscriptionType. func (bm *BroadcastManager) clientCountByType(subType SubscriptionType) (count int64) { bm.clientLock.RLock() defer bm.clientLock.RUnlock() for _, c := range bm.clients { if c.subType == subType { count++ } } return count } func (bm *BroadcastManager) GetSkippedCerts() map[string]uint64 { bm.clientLock.RLock() defer bm.clientLock.RUnlock() skippedCerts := make(map[string]uint64, len(bm.clients)) for _, c := range bm.clients { skippedCerts[c.name] = c.skippedCerts } return skippedCerts } // broadcaster is run in a goroutine and handles the dispatching of entries to clients. func (bm *BroadcastManager) broadcaster() { for { entry := <-bm.Broadcast dataLite := entry.JSONLite() dataFull := entry.JSON() dataDomain := entry.JSONDomains() var data []byte bm.clientLock.RLock() for _, c := range bm.clients { switch c.subType { case SubTypeLite: data = dataLite case SubTypeFull: data = dataFull case SubTypeDomain: data = dataDomain default: log.Printf("Unknown subscription type '%d' for client '%s'. Skipping this client!\n", c.subType, c.name) continue } select { case c.broadcastChan <- data: default: // Default case is executed if the client's broadcast channel is full. c.skippedCerts++ if c.skippedCerts%1000 == 1 { log.Printf("Not providing client '%s' with cert because client's buffer is full. The client can't keep up. Skipped certs: %d\n", c.name, c.skippedCerts) } } } bm.clientLock.RUnlock() } } certstream-server-go-1.7.0/internal/web/client.go000066400000000000000000000073201473673250000220060ustar00rootroot00000000000000package web import ( "log" "strings" "time" "github.com/gorilla/websocket" ) const ( SubTypeFull SubscriptionType = iota SubTypeLite SubTypeDomain ) type SubscriptionType int // client represents a single client's connection to the server. type client struct { conn *websocket.Conn broadcastChan chan []byte name string subType SubscriptionType skippedCerts uint64 } func newClient(conn *websocket.Conn, subType SubscriptionType, name string, certBufferSize int) *client { return &client{ conn: conn, broadcastChan: make(chan []byte, certBufferSize), name: name, subType: subType, } } // Each client has a broadcastHandler that runs in the background and sends out the broadcast messages to the client. func (c *client) broadcastHandler() { writeWait := 60 * time.Second pingTicker := time.NewTicker(30 * time.Second) defer func() { log.Println("Closing broadcast handler for client:", c.conn.RemoteAddr()) pingTicker.Stop() _ = c.conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) _ = c.conn.WriteMessage(websocket.CloseMessage, []byte{}) _ = c.conn.Close() }() for { select { case <-pingTicker.C: _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { return } case message := <-c.broadcastChan: _ = c.conn.SetWriteDeadline(time.Now().Add(writeWait)) w, err := c.conn.NextWriter(websocket.TextMessage) if err != nil { log.Printf("Error while getting next writer: %v\n", err) return } _, writeErr := w.Write(message) if writeErr != nil { log.Printf("Error while writing: %v\n", writeErr) } if closeErr := w.Close(); closeErr != nil { log.Printf("Error while closing: %v\n", closeErr) return } } } } // listenWebsocket is running in the background on a goroutine and listens for messages from the client. // It responds to ping messages with a pong message. It closes the connection if the client sends // a close message or no ping is received within 65 seconds. func (c *client) listenWebsocket() { defer func() { _ = c.conn.Close() ClientHandler.unregisterClient(c) }() readWait := 65 * time.Second c.conn.SetReadLimit(512) _ = c.conn.SetReadDeadline(time.Now().Add(readWait)) defaultPingHandler := c.conn.PingHandler() c.conn.SetPingHandler(func(appData string) error { // Ping received - reset the deadline err := c.conn.SetReadDeadline(time.Now().Add(readWait)) if err != nil { return err } return defaultPingHandler(appData) }) c.conn.SetPongHandler(func(string) error { // Pong received - reset the deadline err := c.conn.SetReadDeadline(time.Now().Add(readWait)) return err }) // Handle messages from the client for { // ignore any message sent from clients - we only handle errors (aka. disconnects) _, _, readErr := c.conn.ReadMessage() if readErr != nil { if websocket.IsUnexpectedCloseError(readErr, websocket.CloseGoingAway, websocket.CloseNormalClosure) { log.Printf("Unexpected websocket close error: %v\n", readErr) } if strings.Contains(strings.ToLower(readErr.Error()), "i/o timeout") { log.Printf("No ping received from client: %v\n", c.conn.RemoteAddr()) closeMessage := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "No ping received!") c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(5*time.Second)) //nolint:errcheck } else if strings.Contains(strings.ToLower(readErr.Error()), "an existing connection was forcibly closed by the remote host") { log.Printf("Connection to client lost: %v\n", c.conn.RemoteAddr()) } log.Printf("Disconnecting client %v!\n", c.conn.RemoteAddr()) break } } } certstream-server-go-1.7.0/internal/web/examplecert.go000066400000000000000000000021311473673250000230340ustar00rootroot00000000000000package web import ( "net/http" "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" ) var exampleCert certstream.Entry // exampleFull handles requests to the /full-stream/example.json endpoint. // It returns a JSON representation of the full example certificate. func exampleFull(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(exampleCert.JSON()) //nolint:errcheck } // exampleLite handles requests to the /example.json endpoint. // It returns a JSON representation of the lite example certificate. func exampleLite(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(exampleCert.JSONLite()) //nolint:errcheck } // exampleDomains handles requests to the /domains-only/example.json endpoint. // It returns a JSON representation of the domain data. func exampleDomains(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(exampleCert.JSONDomains()) //nolint:errcheck } func SetExampleCert(cert certstream.Entry) { exampleCert = cert } certstream-server-go-1.7.0/internal/web/server.go000066400000000000000000000207501473673250000220400ustar00rootroot00000000000000package web import ( "crypto/tls" "fmt" "io" "log" "net" "net/http" "time" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/d-Rickyy-b/certstream-server-go/internal/certstream" "github.com/d-Rickyy-b/certstream-server-go/internal/config" "github.com/gorilla/websocket" ) var ( ClientHandler = BroadcastManager{} upgrader websocket.Upgrader ) type WebServer struct { networkIf string port int routes *chi.Mux server *http.Server certPath string keyPath string } // RegisterPrometheus registers a new handler that listens on the given url and calls the given function // in order to provide metrics for a prometheus server. This function signature was used, because VictoriaMetrics // offers exactly this function signature. func (ws *WebServer) RegisterPrometheus(url string, callback func(w io.Writer, exposeProcessMetrics bool)) { ws.routes.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { callback(w, config.AppConfig.Prometheus.ExposeSystemMetrics) }) } // IPWhitelist returns a middleware that checks if the IP of the client is in the whitelist. func IPWhitelist(whitelist []string) func(next http.Handler) http.Handler { // build a list of whitelisted IPs and CIDRs log.Println("Building IP whitelist...") var ipList []net.IP var cidrList []net.IPNet for _, element := range whitelist { ip, ipNet, err := net.ParseCIDR(element) if err != nil { if ip = net.ParseIP(element); ip == nil { log.Println("Invalid IP in metrics whitelist: ", element) continue } ipList = append(ipList, ip) continue } cidrList = append(cidrList, *ipNet) } log.Println("IP whitelist: ", ipList) log.Println("CIDR whitelist: ", cidrList) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // if the whitelist is empty, just continue if len(ipList) == 0 && len(cidrList) == 0 { next.ServeHTTP(w, r) return } ipString, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { http.Error(w, "InternalServerError", http.StatusInternalServerError) return } ip := net.ParseIP(ipString) for _, cidr := range cidrList { if cidr.Contains(ip) { next.ServeHTTP(w, r) return } } for _, whitelistedIP := range ipList { if whitelistedIP.Equal(ip) { next.ServeHTTP(w, r) return } } log.Printf("IP %s not in whitelist, rejecting request\n", r.RemoteAddr) http.Error(w, "Forbidden", http.StatusForbidden) return }) } } // initFullWebsocket is called when a client connects to the /full-stream endpoint. // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. func initFullWebsocket(w http.ResponseWriter, r *http.Request) { connection, err := upgradeConnection(w, r) if err != nil { log.Println("Error while trying to upgrade connection:", err) return } setupClient(connection, SubTypeFull, r.RemoteAddr) } // initLiteWebsocket is called when a client connects to the / endpoint. // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. func initLiteWebsocket(w http.ResponseWriter, r *http.Request) { connection, err := upgradeConnection(w, r) if err != nil { log.Println("Error while trying to upgrade connection:", err) return } setupClient(connection, SubTypeLite, r.RemoteAddr) } // initDomainWebsocket is called when a client connects to the /domains-only endpoint. // It upgrades the connection to a websocket and starts a goroutine to listen for messages from the client. func initDomainWebsocket(w http.ResponseWriter, r *http.Request) { connection, err := upgradeConnection(w, r) if err != nil { log.Println("Error while trying to upgrade connection:", err) return } setupClient(connection, SubTypeDomain, r.RemoteAddr) } // upgradeConnection upgrades the connection to a websocket and returns the connection. func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error) { var remoteAddr string xForwardedFor := r.Header.Get("X-Forwarded-For") if xForwardedFor != "" { remoteAddr = fmt.Sprintf("'%s' (X-Forwarded-For: '%s')", r.RemoteAddr, xForwardedFor) } else { remoteAddr = fmt.Sprintf("'%s'", r.RemoteAddr) } log.Printf("Starting new websocket for %s - %s\n", remoteAddr, r.URL) connection, err := upgrader.Upgrade(w, r, nil) if err != nil { return nil, err } defaultCloseHandler := connection.CloseHandler() connection.SetCloseHandler(func(code int, text string) error { log.Printf("Stopping websocket for %s - %s\n", remoteAddr, r.URL) return defaultCloseHandler(code, text) }) return connection, nil } // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) { c := newClient(connection, subscriptionType, name, 300) go c.broadcastHandler() go c.listenWebsocket() ClientHandler.registerClient(c) } // setupWebsocketRoutes configures all the routes necessary for the websocket webserver. func setupWebsocketRoutes(r *chi.Mux) { r.Use(middleware.Recoverer) r.Route("/", func(r chi.Router) { r.Route(config.AppConfig.Webserver.FullURL, func(r chi.Router) { r.HandleFunc("/", initFullWebsocket) r.HandleFunc("/example.json", exampleFull) }) r.Route(config.AppConfig.Webserver.LiteURL, func(r chi.Router) { r.HandleFunc("/", initLiteWebsocket) r.HandleFunc("/example.json", exampleLite) }) r.Route(config.AppConfig.Webserver.DomainsOnlyURL, func(r chi.Router) { r.HandleFunc("/", initDomainWebsocket) r.HandleFunc("/example.json", exampleDomains) }) }) } func (ws *WebServer) initServer() { addr := fmt.Sprintf("%s:%d", ws.networkIf, ws.port) tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519}, CipherSuites: []uint16{ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_RSA_WITH_AES_256_CBC_SHA, }, } ws.server = &http.Server{ Addr: addr, Handler: ws.routes, TLSConfig: tlsConfig, IdleTimeout: time.Minute, ReadTimeout: 10 * time.Second, ReadHeaderTimeout: 2 * time.Second, WriteTimeout: 10 * time.Second, } } // NewMetricsServer creates a new webserver that listens on the given port and provides metrics for a metrics server. func NewMetricsServer(networkIf string, port int, certPath, keyPath string) *WebServer { server := &WebServer{ networkIf: networkIf, port: port, routes: chi.NewRouter(), certPath: certPath, keyPath: keyPath, } server.routes.Use(middleware.Recoverer) if config.AppConfig.Prometheus.RealIP { server.routes.Use(middleware.RealIP) } // Enable IP whitelist if configured if len(config.AppConfig.Prometheus.Whitelist) > 0 { server.routes.Use(IPWhitelist(config.AppConfig.Prometheus.Whitelist)) } server.initServer() return server } // NewWebsocketServer starts a new webserver and initialized it with the necessary routes. // It also starts the broadcaster in ClientHandler as a background job and takes care of // setting up websocket.Upgrader. func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *WebServer { server := &WebServer{ networkIf: networkIf, port: port, routes: chi.NewRouter(), certPath: certPath, keyPath: keyPath, } upgrader = websocket.Upgrader{ EnableCompression: config.AppConfig.Webserver.CompressionEnabled, } if config.AppConfig.Webserver.RealIP { server.routes.Use(middleware.RealIP) } // Enable IP whitelist if configured if len(config.AppConfig.Webserver.Whitelist) > 0 { server.routes.Use(IPWhitelist(config.AppConfig.Webserver.Whitelist)) } setupWebsocketRoutes(server.routes) server.initServer() ClientHandler.Broadcast = make(chan certstream.Entry, 10_000) go ClientHandler.broadcaster() return server } // Start initializes the webserver and starts listening for connections. func (ws *WebServer) Start() { addr := fmt.Sprintf("%s:%d", ws.networkIf, ws.port) log.Printf("Starting webserver on %s\n", addr) var err error if ws.keyPath != "" && ws.certPath != "" { err = ws.server.ListenAndServeTLS(ws.certPath, ws.keyPath) } else { err = ws.server.ListenAndServe() } if err != nil { log.Fatal("Error while serving webserver: ", err) } }