pax_global_header00006660000000000000000000000064146047214430014517gustar00rootroot0000000000000052 comment=c3c127b902268c0ec143d5d0e06b567e0b930c45 bypass4netns-0.4.1/000077500000000000000000000000001460472144300141565ustar00rootroot00000000000000bypass4netns-0.4.1/.github/000077500000000000000000000000001460472144300155165ustar00rootroot00000000000000bypass4netns-0.4.1/.github/dependabot.yml000066400000000000000000000004641460472144300203520ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: gomod directory: "/" schedule: interval: daily open-pull-requests-limit: 10 groups: golang-x: patterns: - "golang.org/x/*" - package-ecosystem: github-actions directory: "/" schedule: interval: daily open-pull-requests-limit: 10 bypass4netns-0.4.1/.github/workflows/000077500000000000000000000000001460472144300175535ustar00rootroot00000000000000bypass4netns-0.4.1/.github/workflows/test.yaml000066400000000000000000000237171460472144300214300ustar00rootroot00000000000000--- name: Test on: push: branches: - master - main - release/** pull_request: null workflow_dispatch: jobs: golangci-lint: runs-on: ubuntu-22.04 timeout-minutes: 20 steps: - uses: actions/checkout@v4.1.2 with: fetch-depth: 1 - uses: actions/setup-go@v5 with: go-version: 1.22.x - run: sudo apt-get update && sudo apt-get install -y libseccomp-dev - name: golangci-lint uses: golangci/golangci-lint-action@v4.0.0 with: version: v1.56.2 args: --verbose create-lxc-image: name: create-lxc-image runs-on: ubuntu-22.04 timeout-minutes: 10 steps: - uses: actions/checkout@v4.1.2 - uses: actions/cache/restore@v4 id: cache-restore with: key: lxc-image-base-${{ hashFiles('go.sum', 'test/init_test.sh') }} path: /tmp/test-image.tar.zst lookup-only: true - name: setup lxd id: s1 if: steps.cache-restore.outputs.cache-hit != 'true' run: ./test/setup_lxd.sh - name: launch lxc container id: s2 if: steps.s1.conclusion == 'success' run: ./test/launch_test_lxc.sh - name: install dependencies and build id: s3 if: steps.s2.conclusion == 'success' run: sudo lxc exec test -- sudo --login --user ubuntu /host/test/init_test.sh - name: export image id: s4 if: steps.s3.conclusion == 'success' run: ./test/export_lxc_image.sh test - uses: actions/cache/save@v4 id: s5 if: steps.s4.conclusion == 'success' with: key: lxc-image-base-${{ hashFiles('go.sum', 'test/init_test.sh') }} path: /tmp/test-image.tar.zst test: runs-on: ubuntu-22.04 needs: create-lxc-image timeout-minutes: 20 steps: - uses: actions/checkout@v4.1.2 - name: setup lxd run: ./test/setup_lxd.sh - uses: actions/cache/restore@v4 id: cache-restore with: key: lxc-image-base-${{ hashFiles('go.sum', 'test/init_test.sh') }} path: /tmp/test-image.tar.zst fail-on-cache-miss: true - name: load lxc image run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export - name: launch lxc container run: ./test/launch_test_lxc.sh test-export - name: run test run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/bypass4netns/test/run_test.sh SYNC" # some source codes may be updated. re-export new image. - name: export image run: sudo lxc image alias delete test-export && rm -f /tmp/test-image.tar.zst && ./test/export_lxc_image.sh test - uses: actions/cache/save@v4 with: key: lxc-image-${{ github.sha }} path: /tmp/test-image.tar.zst #- name: debug # run: ./debug.sh benchmark: runs-on: ubuntu-22.04 needs: test timeout-minutes: 20 strategy: matrix: script: ["iperf3/iperf3_host", "iperf3/iperf3", "postgres/postgres", "redis/redis", "block/block", "memcached/memcached", "rabbitmq/rabbitmq", "etcd/etcd", "mysql/mysql"] steps: - uses: actions/checkout@v4.1.2 - name: setup lxd run: ./test/setup_lxd.sh - uses: actions/cache/restore@v4 id: cache-restore with: key: lxc-image-${{ github.sha }} path: /tmp/test-image.tar.zst fail-on-cache-miss: true - name: load lxc image run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export - name: launch lxc container run: ./test/launch_test_lxc.sh test-export - name: run benchmark (${{ matrix.script }}) run: sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sleep 3 && /home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}.sh" - name: upload plot id: get_plot if: matrix.script != 'iperf3/iperf3_host' run: | mkdir /tmp/benchmark-results sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-rootful-direct.log /tmp/benchmark-results/. sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-rootful-host.log /tmp/benchmark-results/. sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-wo-b4ns-direct.log /tmp/benchmark-results/. sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-wo-b4ns-host.log /tmp/benchmark-results/. sudo lxc file pull test/home/ubuntu/bypass4netns/benchmark/${{ matrix.script }}-w-b4ns.log /tmp/benchmark-results/. - uses: actions/upload-artifact@v3 if: steps.get_plot.conclusion == 'success' with: name: benchmark-results path: /tmp/benchmark-results benchmark-multinode: runs-on: ubuntu-22.04 needs: test timeout-minutes: 20 strategy: matrix: script: ["iperf3/iperf3", "postgres/postgres", "redis/redis", "block/block", "memcached/memcached", "rabbitmq/rabbitmq", "etcd/etcd", "mysql/mysql"] steps: - uses: actions/checkout@v4.1.2 - name: setup lxd run: ./test/setup_lxd.sh - uses: actions/cache/restore@v4 id: cache-restore with: key: lxc-image-${{ github.sha }} path: /tmp/test-image.tar.zst fail-on-cache-miss: true - name: load lxc image run: sudo lxc image import /tmp/test-image.tar.zst --alias test-export - name: launch lxc container run: ./test/launch_test_lxc.sh test-export - name: run benchmark (${{ matrix.script }}) run: ./benchmark/${{ matrix.script }}_multinode.sh - name: upload plot run: | mkdir /tmp/benchmark-results cp benchmark/${{ matrix.script }}-multinode-rootful.log /tmp/benchmark-results/. cp benchmark/${{ matrix.script }}-multinode-wo-b4ns.log /tmp/benchmark-results/. cp benchmark/${{ matrix.script }}-multinode-w-b4ns.log /tmp/benchmark-results/. - uses: actions/upload-artifact@v3 with: name: benchmark-results path: /tmp/benchmark-results plot: runs-on: ubuntu-22.04 needs: [benchmark, benchmark-multinode] timeout-minutes: 10 steps: - uses: actions/checkout@v4.1.2 - run: sudo apt update && sudo apt install python3 python3-pip - run: pip3 install matplotlib numpy - uses: actions/download-artifact@v3 with: name: benchmark-results path: ./ - run: mkdir /tmp/benchmark-plots - run: python3 benchmark/redis/redis_plot.py redis-rootful-direct.log redis-rootful-host.log redis-wo-b4ns-direct.log redis-wo-b4ns-host.log redis-w-b4ns.log /tmp/benchmark-plots/redis.png - run: python3 benchmark/redis/redis_plot.py redis-multinode-rootful.log redis-multinode-wo-b4ns.log redis-multinode-w-b4ns.log /tmp/benchmark-plots/redis-multinode.png - run: python3 benchmark/iperf3/iperf3_plot.py iperf3-rootful-direct.log iperf3-rootful-host.log iperf3-wo-b4ns-direct.log iperf3-wo-b4ns-host.log iperf3-w-b4ns.log /tmp/benchmark-plots/iperf3.png - run: python3 benchmark/iperf3/iperf3_plot.py iperf3-multinode-rootful.log iperf3-multinode-wo-b4ns.log iperf3-multinode-w-b4ns.log /tmp/benchmark-plots/iperf3-multinode.png - run: python3 benchmark/postgres/postgres_plot.py postgres-rootful-direct.log postgres-rootful-host.log postgres-wo-b4ns-direct.log postgres-wo-b4ns-host.log postgres-w-b4ns.log /tmp/benchmark-plots/postgres.png - run: python3 benchmark/postgres/postgres_plot.py postgres-multinode-rootful.log postgres-multinode-wo-b4ns.log postgres-multinode-w-b4ns.log /tmp/benchmark-plots/postgres-multinode.png - run: python3 benchmark/block/block_plot.py block-rootful-direct.log block-rootful-host.log block-wo-b4ns-direct.log block-wo-b4ns-host.log block-w-b4ns.log /tmp/benchmark-plots/block.png - run: python3 benchmark/block/block_plot.py block-multinode-rootful.log block-multinode-wo-b4ns.log block-multinode-w-b4ns.log /tmp/benchmark-plots/block-multinode.png - run: python3 benchmark/memcached/memcached_plot.py memcached-rootful-direct.log memcached-rootful-host.log memcached-wo-b4ns-direct.log memcached-wo-b4ns-host.log memcached-w-b4ns.log /tmp/benchmark-plots/memcached.png - run: python3 benchmark/memcached/memcached_plot.py memcached-multinode-rootful.log memcached-multinode-wo-b4ns.log memcached-multinode-w-b4ns.log /tmp/benchmark-plots/memcached-multinode.png - run: python3 benchmark/rabbitmq/rabbitmq_plot.py rabbitmq-rootful-direct.log rabbitmq-rootful-host.log rabbitmq-wo-b4ns-direct.log rabbitmq-wo-b4ns-host.log rabbitmq-w-b4ns.log /tmp/benchmark-plots/rabbitmq.png - run: python3 benchmark/rabbitmq/rabbitmq_plot.py rabbitmq-multinode-rootful.log rabbitmq-multinode-wo-b4ns.log rabbitmq-multinode-w-b4ns.log /tmp/benchmark-plots/rabbitmq-multinode.png - run: python3 benchmark/etcd/etcd_plot.py etcd-rootful-direct.log etcd-rootful-host.log etcd-wo-b4ns-direct.log etcd-wo-b4ns-host.log etcd-w-b4ns.log /tmp/benchmark-plots/etcd.png - run: python3 benchmark/etcd/etcd_plot.py etcd-multinode-rootful.log etcd-multinode-wo-b4ns.log etcd-multinode-w-b4ns.log /tmp/benchmark-plots/etcd-multinode.png - run: python3 benchmark/mysql/mysql_plot.py mysql-rootful-direct.log mysql-rootful-host.log mysql-wo-b4ns-direct.log mysql-wo-b4ns-host.log mysql-w-b4ns.log /tmp/benchmark-plots/mysql.png - run: python3 benchmark/mysql/mysql_plot.py mysql-multinode-rootful.log mysql-multinode-wo-b4ns.log mysql-multinode-w-b4ns.log /tmp/benchmark-plots/mysql-multinode.png - uses: actions/upload-artifact@v3 with: name: benchmark-plots path: /tmp/benchmark-plots bench-script: runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4.1.2 - run: sudo ip a add 192.168.6.2/32 dev eth0 - run: hostname -I - run: ./test/init_test.sh - run: ~/bypass4netns/benchmark/run_bench.sh bypass4netns-0.4.1/.gitignore000066400000000000000000000000521460472144300161430ustar00rootroot00000000000000/bypass4netns /bypass4netnsd *~ /.vagrant bypass4netns-0.4.1/LICENSE000066400000000000000000000261361460472144300151730ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. bypass4netns-0.4.1/Makefile000066400000000000000000000020041460472144300156120ustar00rootroot00000000000000GO ?= go PACKAGE := github.com/rootless-containers/bypass4netns VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags) VERSION_TRIMMED := $(VERSION:v%=%) GO_BUILD_FLAGS += -trimpath GO_BUILD_LDFLAGS += -s -w -X $(PACKAGE)/pkg/version.Version=$(VERSION) GO_BUILD := $(GO) build $(GO_BUILD_FLAGS) -ldflags "$(GO_BUILD_LDFLAGS)" GO_BUILD_STATIC := CGO_ENABLED=1 $(GO) build $(GO_BUILD_FLAGS) -tags "netgo osusergo" -ldflags "$(GO_BUILD_LDFLAGS) -extldflags -static" STRIP ?= strip .DEFAULT: all all: dynamic dynamic: $(GO_BUILD) ./cmd/bypass4netns $(GO_BUILD) ./cmd/bypass4netnsd static: $(GO_BUILD_STATIC) ./cmd/bypass4netns $(GO_BUILD_STATIC) ./cmd/bypass4netnsd strip: $(STRIP) bypass4netns bypass4netnsd install: install bypass4netns /usr/local/bin/bypass4netns install bypass4netnsd /usr/local/bin/bypass4netnsd uninstall: rm -rf /usr/local/bin/bypass4netns /usr/local/bin/bypass4netnsd clean: rm -rf bypass4netns bypass4netnsd .PHONY: all dynamic static strip install uninstall clean bypass4netns-0.4.1/README.md000066400000000000000000000104451460472144300154410ustar00rootroot00000000000000# bypass4netns: Accelerator for slirp4netns using `SECCOMP_IOCTL_NOTIF_ADDFD` (Kernel 5.9) bypass4netns is as fast as `--net=host` and _almost_ as secure as traditional slirp4netns. The current version of bypass4netns needs to be used in conjunction with slirp4netns, however, future version may work without slirp4netns. ## Benchmark ([Oct 16, 2020](https://github.com/rootless-containers/bypass4netns/tree/0f2633f8c8022d39caacd94372855df401411ae2)) Workload: `iperf3 -c HOST_IP` from `podman run` - `--net=host` (insecure): 57.9 Gbps - **bypass4netns**: **56.5 Gbps** - slirp4netns: 7.56 Gbps ## How it works bypass4netns eliminates the overhead of slirp4netns by trapping socket syscals and executing them in the host network namespace using [`SECCOMP_IOCTL_NOTIF_ADDFD`](https://man7.org/linux/man-pages/man2/seccomp_unotify.2.html). See also the [talks](#talks). ## Requirements - kernel >= 5.9 - runc >= 1.1, or crun >= 1.6 - libseccomp >= 2.5 - Rootless Docker, Rootless Podman, or Rootless containerd/nerdctl Build-time requirement: - golang >= 1.17 ## Compile ```console make sudo make install ``` The following binaries will be installed into `/usr/local/bin`: - `bypass4netns`: the bypass4netns binary. - `bypass4netnsd`: an optional [REST](./pkg/api/daemon/openapi.yaml) daemon for controlling bypass4netns processes from a non-initial network namespaces. Used by nerdctl. ## Usage ### Hard way (docker|podman|nerdctl) ```console $ bypass4netns --ignore="127.0.0.0/8,10.0.0.0/8,auto" -p="8080:80" ``` `--ignore=...` is a list of the CIDRs that cannot be bypassed: - loopback CIDRs (`127.0.0.0/8`) - slirp4netns CIDR (`10.0.0.0/8`) - CNI CIDRs inside the slirp's network namespace (`auto`) ```console $ ./test/seccomp.json.sh >$HOME/seccomp.json $ $DOCKER run -it --rm --security-opt seccomp=$HOME/seccomp.json --runtime=runc alpine ``` `$DOCKER` is either `docker`, `podman`, or `nerdctl`. ### Easy way (nerdctl) bypass4netns is experimentally integrated into nerdctl (>= 0.17.0). ```bash containerd-rootless-setuptool.sh install-bypass4netnsd nerdctl run -it --rm -p 8080:80 --annotation nerdctl/bypass4netns=true alpine ``` NOTE: nerdctl prior to v2.0 needs `--label` instead of `--annotation`. Also, the syntax will be probably replaced with `--security-opt` or something like `--network-opt` in a future version of nerdctl. ## :warning: Caveats :warning: Accesses to host abstract sockets and host loopback IPs (127.0.0.0/8) from containers are designed to be rejected. However, it is probably possible to connect to host loopback IPs by exploiting [TOCTOU](https://elixir.bootlin.com/linux/v5.9/source/include/uapi/linux/seccomp.h#L81) of `struct sockaddr *` pointers. ## TODOs - Integration for Docker - Integration for Podman - Enable to connect to port-fowarded ports from other containers - This means that a container with publish option like `-p 8080:80` cannot be connected to port `80` from other containers in the same network namespace - Handle protocol specific publish option like `-p 8080:80/udp`. - Currently, bypass4netns ignores porotocol in publish option. - Bind port when bypass4netns starts with publish option like `-p 8080:80` - Currently, bypass4netns bind socket to port `8080` when it handles bind(2) with target port `80`. - bind(2) can fail if other process bind port `8080` before container's process bind port `80` ## Publications - [Naoki Matsumoto](https://github.com/naoki9911) and [Akihiro Suda](https://github.com/AkihiroSuda). Accelerating TCP/IP Communications in Rootless Containers by Socket Switching. Presented in [_the 156th meeting of the Special Interest Groups on System Software and Operating System (SIGOS)_](http://www.ipsj.or.jp/sig/os/index.php?2022%C7%AF7%B7%EE%B8%A6%B5%E6%B2%F1), SWoPP 2022, Shimonoseki, Japan, July 2022. - [Paper (English)](https://pibvt.net/IPSJ-OS22156009.pdf) ([Copyright notice](https://pibvt.net/notice-ipsj.html)) - [Slides (Japanese)](https://speakerdeck.com/mt2naoki/ip-communications-in-rootless-containers-by-socket-switching) - [Naoki Matsumoto](https://github.com/naoki9911) and [Akihiro Suda](https://github.com/AkihiroSuda). bypass4netns: Accelerating TCP/IP Communications in Rootless Containers. [arXiv:2402.00365 [cs.NI]](https://arxiv.org/abs/2402.00365), February 2024. - [Paper](https://arxiv.org/pdf/2402.00365.pdf) bypass4netns-0.4.1/benchmark/000077500000000000000000000000001460472144300161105ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/.gitignore000066400000000000000000000000301460472144300200710ustar00rootroot00000000000000*.csv *.png *.json *.logbypass4netns-0.4.1/benchmark/block/000077500000000000000000000000001460472144300172025ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/block/.gitignore000066400000000000000000000000131460472144300211640ustar00rootroot00000000000000blk-* benchbypass4netns-0.4.1/benchmark/block/Dockerfile000066400000000000000000000004561460472144300212010ustar00rootroot00000000000000FROM golang:1.22.0 as bench-builder COPY bench.go . # static link RUN CGO_ENABLED=0 go build -o /bench bench.go FROM ubuntu:22.04 RUN apt-get update && apt-get upgrade -y RUN apt-get install -y wget multitime nginx COPY --from=bench-builder /bench /bench CMD ["/bin/bash", "-c", "sleep infinity"] bypass4netns-0.4.1/benchmark/block/bench.go000066400000000000000000000043071460472144300206140ustar00rootroot00000000000000package main import ( "encoding/json" "flag" "fmt" "io" "net/http" "os" "time" ) var ( url = flag.String("url", "http://localhost/blk-1m", "") threadNum = flag.Int("thread-num", 1, "") count = flag.Int("count", 1, "") ) type BenchmarkResult struct { Url string `json:"url"` Count int `json:"count"` TotalElapsedSecond float64 `json:"totalElapsedSecond"` TotalSize int64 `json:"totalSize"` } func main() { flag.Parse() //fmt.Printf("url = %s\n", *url) //fmt.Printf("thread-num = %d\n", *threadNum) //fmt.Printf("count = %d\n", *count) resultsChan := make(chan BenchmarkResult, *count) for i := 0; i < *threadNum; i++ { go bench(*url, *count, resultsChan) } results := []BenchmarkResult{} for i := 0; i < *threadNum; i++ { r := <-resultsChan results = append(results, r) } res, err := json.Marshal(results) if err != nil { fmt.Printf("failed Marshal err=%q", err) panic("error") } fmt.Fprintln(os.Stdout, string(res)) } func bench(url string, count int, resultChan chan BenchmarkResult) { bufferSize := 1024 * 1024 * 128 // 128 MiB buffer := make([]byte, bufferSize) result := BenchmarkResult{ Url: url, Count: count, TotalElapsedSecond: 0, TotalSize: 0, } for i := 0; i < count; i++ { req, err := http.NewRequest("GET", url, nil) if err != nil { fmt.Printf("failed NewRequest err=%q", err) panic("error") } for { start := time.Now() resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Printf("failed Do err=%q, retrying... %d/%d", err, i, count) time.Sleep(100 * time.Millisecond) continue } if resp.StatusCode != 200 { fmt.Printf("unexpected status code %d", resp.StatusCode) panic("error") } else { for { readSize, err := resp.Body.Read(buffer) if err != nil && err != io.EOF { fmt.Printf("failed Copy() err=%q", err) panic("error") } if readSize == 0 { end := time.Now() elapsed := end.Sub(start).Seconds() result.TotalElapsedSecond += elapsed break } result.TotalSize += int64(readSize) } } resp.Body.Close() break } } resultChan <- result } bypass4netns-0.4.1/benchmark/block/block.sh000077500000000000000000000135041460472144300206360ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) IMAGE_NAME="block" COUNT="10" THREAD_NUM="1" source ~/.profile . ../param.bash ./gen_blocks.sh # sometimes fail to pull images # this is workaround # https://github.com/containerd/nerdctl/issues/622 systemctl --user restart containerd sleep 1 systemctl --user restart buildkit sleep 3 systemctl --user status --no-pager containerd systemctl --user status --no-pager buildkit sudo nerdctl build -f ./Dockerfile -t $IMAGE_NAME . nerdctl build -f ./Dockerfile -t $IMAGE_NAME . BLOCK_SIZES=('1k' '32k' '128k' '512k' '1m' '32m' '128m' '512m' '1g') echo "===== Benchmark: block rooful via NetNS =====" ( set +e sudo nerdctl rm -f block-server sudo nerdctl rm -f block-client set -ex sudo nerdctl run -d --name block-server -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" sudo nerdctl run -d --name block-client $IMAGE_NAME sleep infinity sleep 5 SERVER_IP=$(sudo nerdctl exec block-server hostname -i) LOG_NAME="block-rootful-direct.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do sudo nerdctl exec block-client /bench -count 1 -thread-num 1 -url http://$SERVER_IP/blk-$BLOCK_SIZE sudo nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$SERVER_IP/blk-$BLOCK_SIZE >> $LOG_NAME done sudo nerdctl rm -f block-server sudo nerdctl rm -f block-client ) echo "===== Benchmark: block rootful via host =====" ( set +e sudo nerdctl rm -f block-server sudo nerdctl rm -f block-client set -ex sudo nerdctl run -d --name block-server -p 8080:80 -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" sudo nerdctl run -d --name block-client $IMAGE_NAME sleep infinity sleep 5 LOG_NAME="block-rootful-host.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do sudo nerdctl exec block-client /bench -count 1 -thread-num 1 -url http://$HOST_IP:8080/blk-$BLOCK_SIZE sudo nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$HOST_IP:8080/blk-$BLOCK_SIZE >> $LOG_NAME done sudo nerdctl rm -f block-server sudo nerdctl rm -f block-client ) echo "===== Benchmark: block client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f block-server nerdctl rm -f block-client set -ex nerdctl run -d --name block-server -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" nerdctl run -d --name block-client $IMAGE_NAME sleep infinity sleep 5 SERVER_IP=$(nerdctl exec block-server hostname -i) LOG_NAME="block-wo-b4ns-direct.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do nerdctl exec block-client /bench -count 1 -thread-num 1 -url http://$SERVER_IP/blk-$BLOCK_SIZE nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$SERVER_IP/blk-$BLOCK_SIZE >> $LOG_NAME done nerdctl rm -f block-server nerdctl rm -f block-client ) echo "===== Benchmark: block client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f block-server nerdctl rm -f block-client set -ex nerdctl run -d --name block-server -p 8080:80 -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" nerdctl run -d --name block-client $IMAGE_NAME sleep infinity sleep 5 LOG_NAME="block-wo-b4ns-host.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do nerdctl exec block-client /bench -count 1 -thread-num 1 -url http://$HOST_IP:8080/blk-$BLOCK_SIZE nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$HOST_IP:8080/blk-$BLOCK_SIZE >> $LOG_NAME done nerdctl rm -f block-server nerdctl rm -f block-client ) echo "===== Benchmark: block client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f block-server nerdctl rm -f block-client systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d --name block-server -p 8080:80 -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" nerdctl run --annotation nerdctl/bypass4netns=true -d --name block-client $IMAGE_NAME sleep infinity LOG_NAME="block-w-b4ns.log" sleep 5 rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do nerdctl exec block-client /bench -count 1 -thread-num 1 -url http://$HOST_IP:8080/blk-$BLOCK_SIZE nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$HOST_IP:8080/blk-$BLOCK_SIZE >> $LOG_NAME done nerdctl rm -f block-server nerdctl rm -f block-client systemctl --user stop run-bypass4netnsd ) echo "===== Benchmark: block client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f block-server nerdctl rm -f block-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d --name block-server -p 8080:80 -v $(pwd):/var/www/html:ro $IMAGE_NAME nginx -g "daemon off;" nerdctl run --annotation nerdctl/bypass4netns=true -d --name block-client $IMAGE_NAME sleep infinity SERVER_IP=$(nerdctl exec block-server hostname -i) for BLOCK_SIZE in ${BLOCK_SIZES[@]} do nerdctl exec block-client /bench -count $COUNT -thread-num $THREAD_NUM -url http://$SERVER_IP/blk-$BLOCK_SIZE done nerdctl rm -f block-server nerdctl rm -f block-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/block/block_multinode.sh000077500000000000000000000117041460472144300227160ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f block-server NAME="test" exec_lxc nerdctl rm -f block-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" IMAGE_NAME="block" COUNT="10" BLOCK_SIZES=('1k' '32k' '128k' '512k' '1m' '32m' '128m' '512m' '1g') set -eux -o pipefail NAME="test" exec_lxc systemctl --user restart containerd sleep 1 NAME="test" exec_lxc systemctl --user restart buildkit sleep 3 NAME="test" exec_lxc systemctl --user status --no-pager containerd NAME="test" exec_lxc systemctl --user status --no-pager buildkit NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/block && sudo nerdctl build -f ./Dockerfile -t $IMAGE_NAME ." NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/block && nerdctl build -f ./Dockerfile -t $IMAGE_NAME ." sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') NAME="test" exec_lxc /home/ubuntu/bypass4netns/benchmark/block/gen_blocks.sh echo "===== Benchmark: block rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name block-server -v /home/ubuntu/bypass4netns/benchmark/block:/var/www/html:ro $IMAGE_NAME nginx -g \"daemon off;\"" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh block-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name block-client $IMAGE_NAME sleep infinity" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh block-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR LOG_NAME="block-multinode-rootful.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do NAME="test2" exec_lxc /bin/bash -c "sudo nerdctl exec block-client /bench -count $COUNT -thread-num 1 -url http://$TEST1_VXLAN_ADDR/blk-$BLOCK_SIZE" >> $LOG_NAME done NAME="test" exec_lxc sudo nerdctl rm -f block-server NAME="test2" exec_lxc sudo nerdctl rm -f block-client ) echo "===== Benchmark: block client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name block-server -v /home/ubuntu/bypass4netns/benchmark/block:/var/www/html:ro $IMAGE_NAME nginx -g \"daemon off;\"" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh block-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name block-client $IMAGE_NAME sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh block-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR LOG_NAME="block-multinode-wo-b4ns.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do NAME="test2" exec_lxc /bin/bash -c "nerdctl exec block-client /bench -count $COUNT -thread-num 1 -url http://$TEST1_VXLAN_ADDR/blk-$BLOCK_SIZE" >> $LOG_NAME done NAME="test" exec_lxc nerdctl rm -f block-server NAME="test2" exec_lxc nerdctl rm -f block-client ) echo "===== Benchmark: block client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -p 8080:80 -d --name block-server -v /home/ubuntu/bypass4netns/benchmark/block:/var/www/html:ro $IMAGE_NAME nginx -g \"daemon off;\"" SERVER_IP=$(NAME="test" exec_lxc nerdctl exec block-server hostname -i) NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name block-client $IMAGE_NAME sleep infinity" LOG_NAME="block-multinode-w-b4ns.log" rm -f $LOG_NAME for BLOCK_SIZE in ${BLOCK_SIZES[@]} do NAME="test2" exec_lxc /bin/bash -c "nerdctl exec block-client /bench -count $COUNT -thread-num 1 -url http://$SERVER_IP/blk-$BLOCK_SIZE" >> $LOG_NAME done NAME="test" exec_lxc nerdctl rm -f block-server NAME="test2" exec_lxc nerdctl rm -f block-client ) bypass4netns-0.4.1/benchmark/block/block_plot.py000066400000000000000000000023011460472144300217000ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import json import sys BAR_WIDTH=0.25 def load_data(filename): data = {} with open(filename) as f: line = f.readline() while line: for l in json.loads(line): gbps = l["totalSize"] * 8 / l["totalElapsedSecond"] / 1024 / 1024 / 1024 file = l["url"].split("/")[3] if file not in data: data[file] = 0.0 data[file] += gbps line = f.readline() return data labels=['blk-1k', 'blk-32k', 'blk-512k', 'blk-1m', 'blk-32m', 'blk-128m', 'blk-512m', 'blk-1g'] plt.ylabel("Gbps") data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) value = [] for l in labels: value.append(data[l]) plt.bar([x*factor+(BAR_WIDTH*i) for x in range(0, len(labels))], value, align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) plt.legend() plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/block/gen_blocks.sh000077500000000000000000000005321460472144300216470ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) BLOCK_SIZES=(1 32 128 512) for BLOCK_SIZE in ${BLOCK_SIZES[@]} do dd if=/dev/urandom of=blk-${BLOCK_SIZE}k bs=1024 count=$BLOCK_SIZE done for BLOCK_SIZE in ${BLOCK_SIZES[@]} do dd if=/dev/urandom of=blk-${BLOCK_SIZE}m bs=1048576 count=$BLOCK_SIZE done dd if=/dev/urandom of=blk-1g bs=1048576 count=1024 bypass4netns-0.4.1/benchmark/etcd/000077500000000000000000000000001460472144300170275ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/etcd/Dockerfile000066400000000000000000000006421460472144300210230ustar00rootroot00000000000000FROM golang:1.22.0 as bench-builder RUN mkdir /etcd RUN curl -fsSL https://github.com/etcd-io/etcd/archive/refs/tags/v3.5.10.tar.gz | tar -xz --no-same-owner --no-same-permissions --strip-components 1 -C /etcd RUN ls -l /etcd # static link RUN cd /etcd/tools/benchmark && CGO_ENABLED=0 go build -o /bench main.go FROM ubuntu:22.04 COPY --from=bench-builder /bench /bench CMD ["/bin/bash", "-c", "sleep infinity"] bypass4netns-0.4.1/benchmark/etcd/etcd.sh000077500000000000000000000131431460472144300203070ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) ETCD_VERSION="v3.3.25" ETCD_IMAGE="quay.io/coreos/etcd:${ETCD_VERSION}" BENCH_IMAGE="etcd-bench" source ~/.profile . ../param.bash # sometimes fail to pull images # this is workaround # https://github.com/containerd/nerdctl/issues/622 systemctl --user restart containerd sleep 1 systemctl --user restart buildkit sleep 3 systemctl --user status --no-pager containerd systemctl --user status --no-pager buildkit sudo nerdctl build -f ./Dockerfile -t $BENCH_IMAGE . nerdctl build -f ./Dockerfile -t $BENCH_IMAGE . sudo nerdctl pull --quiet $ETCD_IMAGE nerdctl pull --quiet $ETCD_IMAGE echo "===== Benchmark: etcd rootful via NetNS =====" ( set +e sudo nerdctl rm -f etcd-server sudo nerdctl rm -f etcd-client systemctl --user stop etcd-server systemctl --user reset-failed set -ex sudo nerdctl run -d --name etcd-server $ETCD_IMAGE /bin/sh -c "sleep infinity" SERVER_IP=$(sudo nerdctl exec etcd-server hostname -i) systemd-run --user --unit etcd-server sudo nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$SERVER_IP:2379 sleep 5 LOG_NAME="etcd-rootful-direct.log" sudo nerdctl run --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $SERVER_IP:2379 > $LOG_NAME sudo nerdctl rm -f etcd-server sudo nerdctl rm -f etcd-client systemctl --user stop etcd-server systemctl --user reset-failed ) echo "===== Benchmark: etcd rootful via host =====" ( set +e sudo nerdctl rm -f etcd-server sudo nerdctl rm -f etcd-client set -ex sudo nerdctl run -d --name etcd-server -p 12379:2379 $ETCD_IMAGE /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 sleep 5 LOG_NAME="etcd-rootful-host.log" sudo nerdctl run --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $HOST_IP:12379 > $LOG_NAME sudo nerdctl rm -f etcd-server sudo nerdctl rm -f etcd-client ) echo "===== Benchmark: etcd client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop etcd-server systemctl --user reset-failed set -ex nerdctl run -d --name etcd-server $ETCD_IMAGE /bin/sh -c "sleep infinity" SERVER_IP=$(nerdctl exec etcd-server hostname -i) systemd-run --user --unit etcd-server nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$SERVER_IP:2379 sleep 5 LOG_NAME="etcd-wo-b4ns-direct.log" nerdctl run --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $SERVER_IP:2379 > $LOG_NAME nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop etcd-server systemctl --user reset-failed ) echo "===== Benchmark: etcd client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f etcd-server nerdctl rm -f etcd-client set -ex nerdctl run -d --name etcd-server -p 12379:2379 $ETCD_IMAGE /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 sleep 5 LOG_NAME="etcd-wo-b4ns-host.log" nerdctl run --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $HOST_IP:12379 > $LOG_NAME nerdctl rm -f etcd-server nerdctl rm -f etcd-client ) echo "===== Benchmark: etcd client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop run-bypass4netnsd systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d --name etcd-server -p 12379:2379 $ETCD_IMAGE /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 sleep 5 LOG_NAME="etcd-w-b4ns.log" nerdctl run --annotation nerdctl/bypass4netns=true --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $HOST_IP:12379 > $LOG_NAME nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop run-bypass4netnsd systemctl --user reset-failed ) echo "===== Benchmark: etcd client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d --name etcd-server -p 12379:2379 $ETCD_IMAGE /bin/sh -c "sleep infinity" sleep 5 SERVER_IP=$(nerdctl exec etcd-server hostname -i) systemd-run --user --unit etcd-server nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$SERVER_IP:2379 nerdctl run --annotation nerdctl/bypass4netns=true --rm $BENCH_IMAGE /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $SERVER_IP:2379 nerdctl rm -f etcd-server nerdctl rm -f etcd-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/etcd/etcd_multinode.sh000077500000000000000000000127631460472144300223760ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f etcd-server NAME="test" exec_lxc nerdctl rm -f etcd-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" ETCD_VERSION="v3.3.25" ETCD_IMAGE="quay.io/coreos/etcd:${ETCD_VERSION}" BENCH_IMAGE="etcd-bench" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $ETCD_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $ETCD_IMAGE NAME="test" exec_lxc systemctl --user restart containerd sleep 1 NAME="test" exec_lxc systemctl --user restart buildkit sleep 3 NAME="test" exec_lxc systemctl --user status --no-pager containerd NAME="test" exec_lxc systemctl --user status --no-pager buildkit NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/etcd && sudo nerdctl build -f ./Dockerfile -t $BENCH_IMAGE ." NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/etcd && nerdctl build -f ./Dockerfile -t $BENCH_IMAGE ." sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: etcd rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name etcd-server -d $ETCD_IMAGE /bin/sh -c 'sleep infinity'" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh etcd-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test" exec_lxc systemd-run --user --unit etcd-server sudo nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$TEST1_VXLAN_ADDR:2379 NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name etcd-client -d $BENCH_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh etcd-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="etcd-multinode-rootful.log" NAME="test2" exec_lxc sudo nerdctl exec etcd-client /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $TEST1_VXLAN_ADDR:2379 > $LOG_NAME NAME="test" exec_lxc sudo nerdctl rm -f etcd-server NAME="test" exec_lxc systemctl --user stop etcd-server NAME="test" exec_lxc systemctl --user reset-failed NAME="test2" exec_lxc sudo nerdctl rm -f etcd-client ) echo "===== Benchmark: etcd client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name etcd-server -d $ETCD_IMAGE /bin/sh -c 'sleep infinity'" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh etcd-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test" exec_lxc systemd-run --user --unit etcd-server nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$TEST1_VXLAN_ADDR:2379 NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name etcd-client -d $BENCH_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh etcd-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="etcd-multinode-wo-b4ns.log" NAME="test2" exec_lxc nerdctl exec etcd-client /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $TEST1_VXLAN_ADDR:2379 > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f etcd-server NAME="test" exec_lxc systemctl --user stop etcd-server NAME="test" exec_lxc systemctl --user reset-failed NAME="test2" exec_lxc nerdctl rm -f etcd-client ) echo "===== Benchmark: etcd client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -p 12379:2379 --name etcd-server -d $ETCD_IMAGE /bin/sh -c 'sleep infinity'" SERVER_IP=$(NAME="test" exec_lxc nerdctl exec etcd-server hostname -i) NAME="test" exec_lxc systemd-run --user --unit etcd-server nerdctl exec etcd-server /usr/local/bin/etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$SERVER_IP:2379 NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true --name etcd-client -d $BENCH_IMAGE /bin/sh -c 'sleep infinity'" sleep 5 LOG_NAME="etcd-multinode-w-b4ns.log" NAME="test2" exec_lxc nerdctl exec etcd-client /bench put --key-size=8 --val-size=256 --conns=10 --clients=10 --total=100000 --endpoints $SERVER_IP:2379 > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f etcd-server NAME="test2" exec_lxc nerdctl rm -f etcd-client )bypass4netns-0.4.1/benchmark/etcd/etcd_plot.py000066400000000000000000000027531460472144300213650ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import csv import sys def load_data(filename): data = {} with open(filename) as f: line = f.readline() while line: line = line.strip().replace("\t", " ") if "Requests/sec" in line: data["Requests/sec"] = float(line.split(" ")[1]) if "Average" in line: data["Latency(ms)"] = float(line.split(" ")[1]) * 1000 line = f.readline() return data BAR_WIDTH=0.25 labels=['Requests/sec', 'Latency(ms)'] data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH datas = [] for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) datas.append(data) fig = plt.figure() ax1 = fig.add_subplot() ax1.set_ylabel("Requests / second") ax2 = ax1.twinx() ax2.set_ylabel("Average latency (ms)") for i in range(0, data_num): filename = sys.argv[1+i] ax1.bar([BAR_WIDTH*i], datas[i][labels[0]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) for i in range(0, data_num): filename = sys.argv[1+i] ax2.bar([factor+BAR_WIDTH*i], datas[i][labels[1]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) h1, l1 = ax1.get_legend_handles_labels() ax1.legend(h1, l1, loc="upper left") plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/iperf3/000077500000000000000000000000001460472144300173005ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/iperf3/iperf3.sh000077500000000000000000000137631460472144300210410ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) ALPINE_IMAGE="public.ecr.aws/docker/library/alpine:3.16" source ~/.profile . ../param.bash sudo nerdctl pull --quiet $ALPINE_IMAGE nerdctl pull --quiet $ALPINE_IMAGE echo "===== Benchmark: iperf3 rootful via NetNS =====" ( set +e sudo nerdctl rm -f iperf3-server sudo nerdctl rm -f iperf3-client systemctl --user stop iperf3-server set -ex sudo nerdctl run -d --name iperf3-server $ALPINE_IMAGE sleep infinity sudo nerdctl exec iperf3-server apk add --no-cache iperf3 sudo nerdctl run -d --name iperf3-client $ALPINE_IMAGE sleep infinity sudo nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server sudo nerdctl exec iperf3-server iperf3 -s SERVER_IP=$(sudo nerdctl exec iperf3-server hostname -i) sleep 1 sudo nerdctl exec iperf3-client iperf3 -c $SERVER_IP -i 0 --connect-timeout 1000 -J > iperf3-rootful-direct.log sudo nerdctl rm -f iperf3-server sudo nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed ) echo "===== Benchmark: iperf3 rootful via host =====" ( set +e sudo nerdctl rm -f iperf3-server sudo nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed set -ex sudo nerdctl run -d --name iperf3-server -p 5202:5201 $ALPINE_IMAGE sleep infinity sudo nerdctl exec iperf3-server apk add --no-cache iperf3 sudo nerdctl run -d --name iperf3-client $ALPINE_IMAGE sleep infinity sudo nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server sudo nerdctl exec iperf3-server iperf3 -s sleep 1 sudo nerdctl exec iperf3-client iperf3 -c $HOST_IP -p 5202 -i 0 --connect-timeout 1000 -J > iperf3-rootful-host.log sudo nerdctl rm -f iperf3-server sudo nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed ) echo "===== Benchmark: iperf3 client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed set -ex nerdctl run -d --name iperf3-server $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-server apk add --no-cache iperf3 nerdctl run -d --name iperf3-client $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s SERVER_IP=$(nerdctl exec iperf3-server hostname -i) sleep 1 nerdctl exec iperf3-client iperf3 -c $SERVER_IP -i 0 --connect-timeout 1000 -J > iperf3-wo-b4ns-direct.log nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed ) echo "===== Benchmark: iperf3 client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed set -ex nerdctl run -d --name iperf3-server -p 5202:5201 $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-server apk add --no-cache iperf3 nerdctl run -d --name iperf3-client $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s sleep 1 nerdctl exec iperf3-client iperf3 -c $HOST_IP -p 5202 -i 0 --connect-timeout 1000 -J > iperf3-wo-b4ns-host.log nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user reset-failed ) echo "===== Benchmark: iperf3 client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user stop run-bypass4netnsd systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d --name iperf3-server -p 5202:5201 $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-server apk add --no-cache iperf3 nerdctl run --annotation nerdctl/bypass4netns=true -d --name iperf3-client $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s sleep 1 nerdctl exec iperf3-client iperf3 -c $HOST_IP -p 5202 -i 0 --connect-timeout 1000 -J > iperf3-w-b4ns.log nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user stop run-bypass4netnsd systemctl --user reset-failed ) echo "===== Benchmark: iperf3 client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d --name iperf3-server -p 5202:5201 $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-server apk add --no-cache iperf3 nerdctl run --annotation nerdctl/bypass4netns=true -d --name iperf3-client $ALPINE_IMAGE sleep infinity nerdctl exec iperf3-client apk add --no-cache iperf3 systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s SERVER_IP=$(nerdctl exec iperf3-server hostname -i) sleep 1 nerdctl exec iperf3-client iperf3 -c $SERVER_IP -i 0 --connect-timeout 1000 nerdctl rm -f iperf3-server nerdctl rm -f iperf3-client systemctl --user stop iperf3-server systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/iperf3/iperf3_host.sh000077500000000000000000000037321460472144300220710ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) source ~/.profile . ../param.bash ALPINE_IMAGE="public.ecr.aws/docker/library/alpine:3.16" nerdctl pull --quiet "${ALPINE_IMAGE}" systemd-run --user --unit run-iperf3 iperf3 -s echo "===== Benchmark: netns -> host With bypass4netns =====" ( set +e nerdctl rm -f test systemctl --user stop run-bypass4netnsd systemctl --user reset-failed set -ex # start bypass4netnsd for nerdctl integration systemd-run --user --unit run-bypass4netnsd bypass4netnsd sleep 1 nerdctl run --annotation nerdctl/bypass4netns=true -d --name test "${ALPINE_IMAGE}" sleep infinity nerdctl exec test apk add --no-cache iperf3 nerdctl exec test iperf3 -c $HOST_IP nerdctl rm -f test ) echo "===== Benchmark: netns -> host Without bypass4netns (for comparison) =====" ( set +e nerdctl rm -f test set -ex nerdctl run -d --name test "${ALPINE_IMAGE}" sleep infinity nerdctl exec test apk add --no-cache iperf3 nerdctl exec test iperf3 -c $HOST_IP nerdctl rm -f test ) echo "===== Benchmark: host -> netns With bypass4netns =====" ( set +e nerdctl rm -f test systemctl --user stop run-iperf3-netns systemctl --user reset-failed set -ex nerdctl run --annotation nerdctl/bypass4netns=true -d --name test -p 8080:5201 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test apk add --no-cache iperf3 systemd-run --user --unit run-iperf3-netns nerdctl exec test iperf3 -s -4 sleep 1 # waiting `iperf3 -s -4` becomes ready iperf3 -c $HOST_IP -p 8080 nerdctl rm -f test ) echo "===== Benchmark: host -> netns Without bypass4netns (for comparison) =====" ( set +e nerdctl rm -f test systemctl --user stop run-iperf3-netns2 systemctl --user reset-failed set -ex nerdctl run -d --name test -p 8080:5201 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test apk add --no-cache iperf3 systemd-run --user --unit run-iperf3-netns2 nerdctl exec test iperf3 -s -4 sleep 1 iperf3 -c $HOST_IP -p 8080 nerdctl rm -f test ) bypass4netns-0.4.1/benchmark/iperf3/iperf3_multinode.sh000077500000000000000000000113531460472144300231120ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f iperf3-server NAME="test" exec_lxc nerdctl rm -f iperf3-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" ALPINE_IMAGE="public.ecr.aws/docker/library/alpine:3.16" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $ALPINE_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $ALPINE_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: iperf3 rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name iperf3-server $ALPINE_IMAGE sleep infinity" NAME="test" exec_lxc sudo nerdctl exec iperf3-server apk add --no-cache iperf3 NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh iperf3-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name iperf3-client $ALPINE_IMAGE sleep infinity" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh iperf3-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR NAME="test2" exec_lxc sudo nerdctl exec iperf3-client apk add --no-cache iperf3 NAME="test" exec_lxc systemd-run --user --unit iperf3-server sudo nerdctl exec iperf3-server iperf3 -s NAME="test2" exec_lxc sudo nerdctl exec iperf3-client iperf3 -c $TEST1_VXLAN_ADDR -i 0 --connect-timeout 1000 -J > iperf3-multinode-rootful.log NAME="test" exec_lxc sudo nerdctl rm -f iperf3-server NAME="test" exec_lxc systemctl --user reset-failed NAME="test2" exec_lxc sudo nerdctl rm -f iperf3-client ) echo "===== Benchmark: iperf3 client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name iperf3-server $ALPINE_IMAGE sleep infinity" NAME="test" exec_lxc nerdctl exec iperf3-server apk add --no-cache iperf3 NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh iperf3-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name iperf3-client $ALPINE_IMAGE sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh iperf3-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR NAME="test2" exec_lxc nerdctl exec iperf3-client apk add --no-cache iperf3 NAME="test" exec_lxc systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s NAME="test2" exec_lxc nerdctl exec iperf3-client iperf3 -c $TEST1_VXLAN_ADDR -i 0 --connect-timeout 1000 -J > iperf3-multinode-wo-b4ns.log NAME="test" exec_lxc nerdctl rm -f iperf3-server NAME="test" exec_lxc systemctl --user reset-failed NAME="test2" exec_lxc nerdctl rm -f iperf3-client ) echo "===== Benchmark: iperf3 client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d -p 5202:5201 --name iperf3-server $ALPINE_IMAGE sleep infinity" NAME="test" exec_lxc nerdctl exec iperf3-server apk add --no-cache iperf3 NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name iperf3-client $ALPINE_IMAGE sleep infinity" NAME="test2" exec_lxc nerdctl exec iperf3-client apk add --no-cache iperf3 SERVER_IP=$(NAME="test" exec_lxc nerdctl exec iperf3-server hostname -i) NAME="test" exec_lxc systemd-run --user --unit iperf3-server nerdctl exec iperf3-server iperf3 -s NAME="test2" exec_lxc nerdctl exec iperf3-client iperf3 -c $SERVER_IP -i 0 --connect-timeout 1000 -J > iperf3-multinode-w-b4ns.log NAME="test" exec_lxc nerdctl rm -f iperf3-server NAME="test2" exec_lxc nerdctl rm -f iperf3-client )bypass4netns-0.4.1/benchmark/iperf3/iperf3_plot.py000066400000000000000000000015471460472144300221070ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import json import sys BAR_WIDTH=0.4 def load_data(filename): with open(filename) as f: return json.load(f) labels=['sum_received.bits_per_second'] #plt.rcParams["figure.figsize"] = (20,4) plt.ylabel("Gbps") data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH for i in range(0, data_num): filename = sys.argv[1+i] data_json = load_data(filename) value = [data_json["end"]["sum_received"]["bits_per_second"] / 1024 / 1024 / 1024] plt.bar([x*factor+(BAR_WIDTH*i) for x in range(0, len(labels))], value, align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) plt.legend() plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/memcached/000077500000000000000000000000001460472144300200165ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/memcached/memcached.sh000077500000000000000000000120111460472144300222560ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) MEMCACHED_VERSION=1.6.22 MEMCACHED_IMAGE="memcached:${MEMCACHED_VERSION}" MEMTIRE_VERSION=2.0.0 MEMTIRE_IMAGE="redislabs/memtier_benchmark:${MEMTIRE_VERSION}" source ~/.profile . ../param.bash sudo nerdctl pull --quiet $MEMCACHED_IMAGE sudo nerdctl pull --quiet $MEMTIRE_IMAGE nerdctl pull --quiet $MEMCACHED_IMAGE nerdctl pull --quiet $MEMTIRE_IMAGE echo "===== Benchmark: memcached rootful via NetNS =====" ( set +e sudo nerdctl rm -f memcached-server sudo nerdctl rm -f memcached-client set -ex sudo nerdctl run -d --name memcached-server $MEMCACHED_IMAGE SERVER_IP=$(sudo nerdctl exec memcached-server hostname -i) LOG_NAME="memcached-rootful-direct.log" sudo nerdctl run --name memcached-client $MEMTIRE_IMAGE --host=$SERVER_IP --port=11211 --protocol=memcache_binary --json-out-file=/$LOG_NAME > /dev/null sudo nerdctl cp memcached-client:/$LOG_NAME ./$LOG_NAME sudo nerdctl rm -f memcached-server sudo nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached rootful via host =====" ( set +e sudo nerdctl rm -f memcached-server sudo nerdctl rm -f memcached-client set -ex sudo nerdctl run -d --name memcached-server -p 11212:11211 $MEMCACHED_IMAGE LOG_NAME="memcached-rootful-host.log" sudo nerdctl run --name memcached-client $MEMTIRE_IMAGE --host=$HOST_IP --port=11212 --protocol=memcache_binary --json-out-file=/$LOG_NAME > /dev/null sudo nerdctl cp memcached-client:/$LOG_NAME ./$LOG_NAME sudo nerdctl rm -f memcached-server sudo nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f memcached-server nerdctl rm -f memcached-client set -ex nerdctl run -d --name memcached-server $MEMCACHED_IMAGE SERVER_IP=$(nerdctl exec memcached-server hostname -i) LOG_NAME="memcached-wo-b4ns-direct.log" nerdctl run -d --name memcached-client --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c "sleep infinity" nerdctl exec memcached-client memtier_benchmark --host=$SERVER_IP --port=11211 --protocol=memcache_binary --json-out-file=/$LOG_NAME > /dev/null nerdctl cp memcached-client:/$LOG_NAME ./$LOG_NAME nerdctl rm -f memcached-server nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f memcached-server nerdctl rm -f memcached-client set -ex nerdctl run -d --name memcached-server -p 11212:11211 $MEMCACHED_IMAGE LOG_NAME="memcached-wo-b4ns-host.log" nerdctl run -d --name memcached-client --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c "sleep infinity" nerdctl exec memcached-client memtier_benchmark --host=$HOST_IP --port=11212 --protocol=memcache_binary --json-out-file=/$LOG_NAME > /dev/null nerdctl cp memcached-client:/$LOG_NAME ./$LOG_NAME nerdctl rm -f memcached-server nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e nerdctl rm -f memcached-server nerdctl rm -f memcached-client systemctl --user stop run-bypass4netnsd systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d --name memcached-server -p 11212:11211 $MEMCACHED_IMAGE LOG_NAME="memcached-w-b4ns.log" nerdctl run --annotation nerdctl/bypass4netns=true -d --name memcached-client --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c "sleep infinity" nerdctl exec memcached-client memtier_benchmark --host=$HOST_IP --port=11212 --protocol=memcache_binary --json-out-file=/$LOG_NAME > /dev/null nerdctl cp memcached-client:/$LOG_NAME ./$LOG_NAME nerdctl rm -f memcached-server nerdctl rm -f memcached-client systemctl --user stop run-bypass4netnsd systemctl --user reset-failed ) echo "===== Benchmark: memcached client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f memcached-server nerdctl rm -f memcached-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d --name memcached-server -p 11212:11211 $MEMCACHED_IMAGE SERVER_IP=$(nerdctl exec memcached-server hostname -i) nerdctl run --annotation nerdctl/bypass4netns=true -d --name memcached-client --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c "sleep infinity" nerdctl exec memcached-client memtier_benchmark --host=$SERVER_IP --port=11211 --protocol=memcache_binary nerdctl rm -f memcached-server nerdctl rm -f memcached-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/memcached/memcached_multinode.sh000077500000000000000000000112671460472144300243520ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f memcached-server NAME="test" exec_lxc nerdctl rm -f memcached-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" MEMCACHED_VERSION=1.6.22 MEMCACHED_IMAGE="memcached:${MEMCACHED_VERSION}" MEMTIRE_VERSION=2.0.0 MEMTIRE_IMAGE="redislabs/memtier_benchmark:${MEMTIRE_VERSION}" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $MEMCACHED_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $MEMCACHED_IMAGE NAME="test" exec_lxc sudo nerdctl pull --quiet $MEMTIRE_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $MEMTIRE_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: memcached rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name memcached-server -d $MEMCACHED_IMAGE" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh memcached-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name memcached-client -d --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh memcached-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="memcached-multinode-rootful.log" NAME="test2" exec_lxc sudo nerdctl exec memcached-client memtier_benchmark --host=$TEST1_VXLAN_ADDR --port=11211 --protocol=memcache_binary --json-out-file=/$LOG_NAME NAME="test2" exec_lxc sudo nerdctl exec memcached-client cat /$LOG_NAME > $LOG_NAME NAME="test" exec_lxc sudo nerdctl rm -f memcached-server NAME="test2" exec_lxc sudo nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name memcached-server -d $MEMCACHED_IMAGE" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh memcached-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name memcached-client -d --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh memcached-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="memcached-multinode-wo-b4ns.log" NAME="test2" exec_lxc nerdctl exec memcached-client memtier_benchmark --host=$TEST1_VXLAN_ADDR --port=11211 --protocol=memcache_binary --json-out-file=/$LOG_NAME NAME="test2" exec_lxc nerdctl exec memcached-client cat /$LOG_NAME > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f memcached-server NAME="test2" exec_lxc nerdctl rm -f memcached-client ) echo "===== Benchmark: memcached client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -p 11212:11211 --name memcached-server -d $MEMCACHED_IMAGE" NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true --name memcached-client -d --entrypoint '' $MEMTIRE_IMAGE /bin/sh -c 'sleep infinity'" SERVER_IP=$(NAME="test" exec_lxc nerdctl exec memcached-server hostname -i) sleep 5 LOG_NAME="memcached-multinode-w-b4ns.log" NAME="test2" exec_lxc nerdctl exec memcached-client memtier_benchmark --host=$SERVER_IP --port=11211 --protocol=memcache_binary --json-out-file=/$LOG_NAME NAME="test2" exec_lxc nerdctl exec memcached-client cat /$LOG_NAME > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f memcached-server NAME="test2" exec_lxc nerdctl rm -f memcached-client )bypass4netns-0.4.1/benchmark/memcached/memcached_plot.py000066400000000000000000000032761460472144300233440ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import json import sys BAR_WIDTH=0.4 def load_data(filename): data = {} with open(filename) as f: d = json.load(f) if d == None: raise Exception("{} has invalid json format".format(filename)) data["Ops/sec"] = [] data["Ops/sec"].append(d["ALL STATS"]["Sets"]["Ops/sec"]) data["Ops/sec"].append(d["ALL STATS"]["Gets"]["Ops/sec"]) data["Latency"] = [] data["Latency"].append(d["ALL STATS"]["Sets"]["Latency"]) data["Latency"].append(d["ALL STATS"]["Gets"]["Latency"]) return data labels=['Sets(Ops/sec)', 'Gets(Ops/sec)', 'Sets(Latency)', 'Gets(Latency)'] plt.ylabel("Ops / seconds") data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH datas = [] for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) datas.append(data) fig = plt.figure() ax1 = fig.add_subplot() ax1.set_ylabel("Operations / second") ax2 = ax1.twinx() ax2.set_ylabel("latency (ms)") for i in range(0, data_num): filename = sys.argv[1+i] ax1.bar([BAR_WIDTH*i, factor+BAR_WIDTH*i], datas[i]["Ops/sec"], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) for i in range(0, data_num): filename = sys.argv[1+i] ax2.bar([factor*2+BAR_WIDTH*i, factor*3+BAR_WIDTH*i], datas[i]["Latency"], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) h1, l1 = ax1.get_legend_handles_labels() ax1.legend(h1, l1, loc="upper left") plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/mysql/000077500000000000000000000000001460472144300172555ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/mysql/Dockerfile000066400000000000000000000002101460472144300212400ustar00rootroot00000000000000FROM ubuntu:22.04 RUN apt-get update && apt-get upgrade -y RUN apt-get install -y sysbench CMD ["/bin/bash", "-c", "sleep infinity"] bypass4netns-0.4.1/benchmark/mysql/mysql.sh000077500000000000000000000152031460472144300207620ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail MYSQL_VERSION=8.2.0 MYSQL_IMAGE="mysql:$MYSQL_VERSION" BENCH_IMAGE="mysql-bench" source ~/.profile cd $(dirname $0) . ../../test/util.sh . ../param.bash # sometimes fail to pull images # this is workaround # https://github.com/containerd/nerdctl/issues/622 systemctl --user restart containerd sleep 1 systemctl --user restart buildkit sleep 3 systemctl --user status --no-pager containerd systemctl --user status --no-pager buildkit sudo nerdctl build -f ./Dockerfile -t $BENCH_IMAGE . nerdctl build -f ./Dockerfile -t $BENCH_IMAGE . sudo nerdctl pull --quiet $MYSQL_IMAGE nerdctl pull --quiet $MYSQL_IMAGE echo "===== Benchmark: mysql rootful via NetNS =====" ( set +e sudo nerdctl rm -f mysql-server sudo nerdctl rm -f mysql-client set -ex sudo nerdctl run -d --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE sudo nerdctl run -d --name mysql-client $BENCH_IMAGE sleep infinity SERVER_IP=$(sudo nerdctl inspect mysql-server | jq -r .[0].NetworkSettings.Networks.'"unknown-eth0"'.IPAddress) sleep 30 sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-rootful-direct.log sudo nerdctl rm -f mysql-server sudo nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql rootful via host =====" ( set +e sudo nerdctl rm -f mysql-server sudo nerdctl rm -f mysql-client set -ex sudo nerdctl run -d -p 13306:3306 --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE sudo nerdctl run -d --name mysql-client $BENCH_IMAGE sleep infinity sleep 30 sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-rootful-host.log sudo nerdctl rm -f mysql-server sudo nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f mysql-server nerdctl rm -f mysql-client set -ex nerdctl run -d --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE nerdctl run -d --name mysql-client $BENCH_IMAGE sleep infinity SERVER_IP=$(nerdctl inspect mysql-server | jq -r .[0].NetworkSettings.Networks.'"unknown-eth0"'.IPAddress) sleep 30 nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-wo-b4ns-direct.log nerdctl rm -f mysql-server nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f mysql-server nerdctl rm -f mysql-client set -ex nerdctl run -d -p 13306:3306 --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE nerdctl run -d --name mysql-client $BENCH_IMAGE sleep infinity sleep 30 nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-wo-b4ns-host.log nerdctl rm -f mysql-server nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f mysql-server nerdctl rm -f mysql-client systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d -p 13306:3306 --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name mysql-client $BENCH_IMAGE sleep infinity sleep 30 nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$HOST_IP --mysql-port=13306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-w-b4ns.log nerdctl rm -f mysql-server nerdctl rm -f mysql-client systemctl --user stop run-bypass4netnsd ) echo "===== Benchmark: mysql client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f mysql-server nerdctl rm -f mysql-client systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d -p 13306:3306 --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name mysql-client $BENCH_IMAGE sleep infinity SERVER_IP=$(nerdctl inspect mysql-server | jq -r .[0].NetworkSettings.Networks.'"unknown-eth0"'.IPAddress) sleep 30 nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-port=3306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-port=3306 --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run nerdctl rm -f mysql-server nerdctl rm -f mysql-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/mysql/mysql_multinode.sh000077500000000000000000000131001460472144300230340ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f mysql-server NAME="test" exec_lxc nerdctl rm -f mysql-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" MYSQL_VERSION=8.2.0 MYSQL_IMAGE="mysql:$MYSQL_VERSION" BENCH_IMAGE="mysql-bench" set -eux -o pipefail # sometimes fail to pull images # this is workaround # https://github.com/containerd/nerdctl/issues/622 NAME="test" exec_lxc systemctl --user restart containerd sleep 1 NAME="test" exec_lxc systemctl --user restart buildkit sleep 3 NAME="test" exec_lxc systemctl --user status --no-pager containerd NAME="test" exec_lxc systemctl --user status --no-pager buildkit NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/mysql && sudo nerdctl build -f ./Dockerfile -t $BENCH_IMAGE ." NAME="test" exec_lxc /bin/bash -c "cd /home/ubuntu/bypass4netns/benchmark/mysql && nerdctl build -f ./Dockerfile -t $BENCH_IMAGE ." NAME="test" exec_lxc sudo nerdctl pull --quiet $MYSQL_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $MYSQL_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: mysql rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh mysql-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged -d --name mysql-client $BENCH_IMAGE sleep infinity" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh mysql-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 30 NAME="test2" exec_lxc sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$TEST1_VXLAN_ADDR --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare NAME="test2" exec_lxc sudo nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$TEST1_VXLAN_ADDR --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-multinode-rootful.log NAME="test" exec_lxc sudo nerdctl rm -f mysql-server NAME="test2" exec_lxc sudo nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh mysql-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged -d --name mysql-client $BENCH_IMAGE sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh mysql-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 30 NAME="test2" exec_lxc nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$TEST1_VXLAN_ADDR --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare NAME="test2" exec_lxc nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$TEST1_VXLAN_ADDR --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-multinode-wo-b4ns.log NAME="test" exec_lxc nerdctl rm -f mysql-server NAME="test2" exec_lxc nerdctl rm -f mysql-client ) echo "===== Benchmark: mysql client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d -p 13306:3306 --name mysql-server -e MYSQL_ROOT_PASSWORD=pass -e MYSQL_DATABASE=bench $MYSQL_IMAGE" NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name mysql-client $BENCH_IMAGE sleep infinity" SERVER_IP=$(NAME="test" exec_lxc nerdctl inspect mysql-server | jq -r .[0].NetworkSettings.Networks.'"unknown-eth0"'.IPAddress) sleep 30 NAME="test2" exec_lxc nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_common prepare NAME="test2" exec_lxc nerdctl exec mysql-client sysbench --threads=4 --time=60 --mysql-host=$SERVER_IP --mysql-db=bench --mysql-user=root --mysql-password=pass --db-driver=mysql oltp_read_write run > mysql-multinode-w-b4ns.log NAME="test" exec_lxc nerdctl rm -f mysql-server NAME="test2" exec_lxc nerdctl rm -f mysql-client )bypass4netns-0.4.1/benchmark/mysql/mysql_plot.py000066400000000000000000000032211460472144300220300ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import csv import sys import re def load_data(filename): data = {} with open(filename) as f: line = f.readline() while line: line = re.sub('\s+', ' ', line.strip()).split(" ") if "transactions:" in line: data["transactions"] = float(line[2].replace("(", "")) if "queries:" in line: data["queries"] = float(line[2].replace("(", "")) if "avg:" in line: data["latency(ms)"] = float(line[1]) line = f.readline() return data BAR_WIDTH=0.25 labels=['transactions', 'queries', 'latency(ms)'] data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH datas = [] for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) datas.append(data) fig = plt.figure() ax1 = fig.add_subplot() ax1.set_ylabel("Operations / second") ax2 = ax1.twinx() ax2.set_ylabel("Average latency (ms)") for i in range(0, data_num): filename = sys.argv[1+i] ax1.bar([BAR_WIDTH*i, factor+BAR_WIDTH*i], [datas[i][labels[0]], datas[i][labels[1]]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) for i in range(0, data_num): filename = sys.argv[1+i] ax2.bar([factor*2+BAR_WIDTH*i], datas[i][labels[2]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) h1, l1 = ax1.get_legend_handles_labels() ax1.legend(h1, l1, loc="upper left") plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/param.bash000066400000000000000000000002371460472144300200510ustar00rootroot00000000000000HOST_IP_PREFIX="192.168.6." HOST_IP=$(HOST=$(hostname -I); for i in ${HOST[@]}; do echo $i | grep -q $HOST_IP_PREFIX; if [ $? -eq 0 ]; then echo $i; fi; done) bypass4netns-0.4.1/benchmark/postgres/000077500000000000000000000000001460472144300177565ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/postgres/postgres.sh000077500000000000000000000122211460472144300221610ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail POSTGRES_VERSION=16.1 POSTGRES_IMAGE="postgres:$POSTGRES_VERSION" source ~/.profile cd $(dirname $0) . ../../test/util.sh . ../param.bash sudo nerdctl pull --quiet $POSTGRES_IMAGE nerdctl pull --quiet $POSTGRES_IMAGE echo "===== Benchmark: postgresql rootful via NetNS =====" ( set +e sudo nerdctl rm -f psql-server sudo nerdctl rm -f psql-client set -ex sudo nerdctl run -d --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE sudo nerdctl run -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity SERVER_IP=$(sudo nerdctl exec psql-server hostname -i) sleep 5 sudo nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -i postgres sudo nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -t 1000 postgres > postgres-rootful-direct.log sudo nerdctl rm -f psql-server sudo nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql rootful via host =====" ( set +e sudo nerdctl rm -f psql-server sudo nerdctl rm -f psql-client set -ex sudo nerdctl run -d -p 15432:5432 --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE sudo nerdctl run -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity sleep 5 sudo nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -i postgres sudo nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -t 1000 postgres > postgres-rootful-host.log sudo nerdctl rm -f psql-server sudo nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f psql-server nerdctl rm -f psql-client set -ex nerdctl run -d --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE nerdctl run -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity SERVER_IP=$(nerdctl exec psql-server hostname -i) PID=$(nerdctl inspect psql-client | jq '.[0].State.Pid') NAME="psql-client" exec_netns /bin/bash -c "until nc -z $SERVER_IP 5432; do sleep 1; done" nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -i postgres nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -t 1000 postgres > postgres-wo-b4ns-direct.log nerdctl rm -f psql-server nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f psql-server nerdctl rm -f psql-client set -ex nerdctl run -d -p 15432:5432 --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE nerdctl run -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity sleep 5 nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -i postgres nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -t 1000 postgres > postgres-wo-b4ns-host.log nerdctl rm -f psql-server nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f psql-server nerdctl rm -f psql-client systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d -p 15432:5432 --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity PID=$(nerdctl inspect psql-client | jq '.[0].State.Pid') NAME="psql-client" exec_netns /bin/bash -c "until nc -z $HOST_IP 15432; do sleep 1; done" nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -i postgres nerdctl exec psql-client pgbench -h $HOST_IP -p 15432 -U postgres -s 10 -t 1000 postgres > postgres-w-b4ns.log nerdctl rm -f psql-server nerdctl rm -f psql-client systemctl --user stop run-bypass4netnsd ) echo "===== Benchmark: postgres client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f psql-server nerdctl rm -f psql-client systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d -p 15432:5432 --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity SERVER_IP=$(nerdctl exec psql-server hostname -i) sleep 5 nerdctl exec psql-client pgbench -h $SERVER_IP -p 5432 -U postgres -s 10 -i postgres nerdctl exec psql-client pgbench -h $SERVER_IP -p 5432 -U postgres -s 10 -t 1000 postgres nerdctl rm -f psql-server nerdctl rm -f psql-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/postgres/postgres_multinode.sh000077500000000000000000000105011460472144300242400ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f psql-server NAME="test" exec_lxc nerdctl rm -f psql-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" POSTGRES_VERSION=16.1 POSTGRES_IMAGE="postgres:$POSTGRES_VERSION" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $POSTGRES_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $POSTGRES_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: postgresql rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name psql-server -e POSTGRES_PASSWORD=pass -d $POSTGRES_IMAGE" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh psql-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name psql-client -e PGPASSWORD=pass -d $POSTGRES_IMAGE sleep infinity" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh psql-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 NAME="test2" exec_lxc sudo nerdctl exec psql-client pgbench -h $TEST1_VXLAN_ADDR -U postgres -s 10 -i postgres NAME="test2" exec_lxc sudo nerdctl exec psql-client pgbench -h $TEST1_VXLAN_ADDR -U postgres -s 10 -t 1000 postgres > postgres-multinode-rootful.log NAME="test" exec_lxc sudo nerdctl rm -f psql-server NAME="test2" exec_lxc sudo nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name psql-server -e POSTGRES_PASSWORD=pass -d $POSTGRES_IMAGE" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh psql-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name psql-client -e PGPASSWORD=pass -d $POSTGRES_IMAGE sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh psql-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 NAME="test2" exec_lxc nerdctl exec psql-client pgbench -h $TEST1_VXLAN_ADDR -U postgres -s 10 -i postgres NAME="test2" exec_lxc nerdctl exec psql-client pgbench -h $TEST1_VXLAN_ADDR -U postgres -s 10 -t 1000 postgres > postgres-multinode-wo-b4ns.log NAME="test" exec_lxc nerdctl rm -f psql-server NAME="test2" exec_lxc nerdctl rm -f psql-client ) echo "===== Benchmark: postgresql client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d -p 15432:5432 --name psql-server -e POSTGRES_PASSWORD=pass $POSTGRES_IMAGE" NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name psql-client -e PGPASSWORD=pass $POSTGRES_IMAGE sleep infinity" SERVER_IP=$(NAME="test" exec_lxc nerdctl exec psql-server hostname -i) sleep 5 NAME="test2" exec_lxc nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -i postgres NAME="test2" exec_lxc nerdctl exec psql-client pgbench -h $SERVER_IP -U postgres -s 10 -t 1000 postgres > postgres-multinode-w-b4ns.log NAME="test" exec_lxc nerdctl rm -f psql-server NAME="test2" exec_lxc nerdctl rm -f psql-client )bypass4netns-0.4.1/benchmark/postgres/postgres_plot.py000066400000000000000000000027501460472144300232400ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import csv import sys def load_data(filename): data = {} with open(filename) as f: line = f.readline() while line: line = line.strip() if "latency average" in line: data["latency (ms)"] = float(line.split(" ")[3]) if "tps" in line: data["tps"] = float(line.split(" ")[2]) line = f.readline() return data BAR_WIDTH=0.25 labels=['tps', 'latency (ms)'] plt.ylabel("Request / seconds") data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH datas = [] for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) datas.append(data) print(data) fig = plt.figure() ax1 = fig.add_subplot() ax1.set_ylabel("transaction / second") ax2 = ax1.twinx() ax2.set_ylabel("latency (ms)") for i in range(0, data_num): filename = sys.argv[1+i] ax1.bar([BAR_WIDTH*i], datas[i][labels[0]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) for i in range(0, data_num): filename = sys.argv[1+i] ax2.bar([factor+BAR_WIDTH*i], datas[i][labels[1]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) h1, l1 = ax1.get_legend_handles_labels() ax1.legend(h1, l1, loc="upper left") plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/rabbitmq/000077500000000000000000000000001460472144300177115ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/rabbitmq/rabbitmq.sh000077500000000000000000000077231460472144300220620ustar00rootroot00000000000000#!/bin/bash RABBITMQ_VERSION=3.12.10 RABBITMQ_IMAGE="rabbitmq:$RABBITMQ_VERSION" PERF_VERSION="2.20.0" PERF_IMAGE="pivotalrabbitmq/perf-test:$PERF_VERSION" source ~/.profile cd $(dirname $0) . ../param.bash sudo nerdctl pull --quiet $RABBITMQ_IMAGE sudo nerdctl pull --quiet $PERF_IMAGE nerdctl pull --quiet $RABBITMQ_IMAGE nerdctl pull --quiet $PERF_IMAGE echo "===== Benchmark: rabbitmq rootful via NetNS =====" ( set +e sudo nerdctl rm -f rabbitmq-server set -ex sudo nerdctl run -d --name rabbitmq-server $RABBITMQ_IMAGE sleep 10 SERVER_IP=$(sudo nerdctl exec rabbitmq-server hostname -i) LOG_NAME="rabbitmq-rootful-direct.log" sudo nerdctl run --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$SERVER_IP --producers 2 --consumers 2 --time 60 > $LOG_NAME sudo nerdctl rm -f rabbitmq-server ) echo "===== Benchmark: rabbitmq rootful via host =====" ( set +e sudo nerdctl rm -f rabbitmq-server set -ex sudo nerdctl run -d --name rabbitmq-server -p 5673:5672 $RABBITMQ_IMAGE sleep 10 SERVER_IP=$(sudo nerdctl exec rabbitmq-server hostname -i) LOG_NAME="rabbitmq-rootful-host.log" sudo nerdctl run --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$HOST_IP:5673 --producers 2 --consumers 2 --time 60 > $LOG_NAME sudo nerdctl rm -f rabbitmq-server ) echo "===== Benchmark: rabbitmq client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f rabbitmq-server set -ex nerdctl run -d --name rabbitmq-server $RABBITMQ_IMAGE sleep 10 SERVER_IP=$(nerdctl exec rabbitmq-server hostname -i) LOG_NAME="rabbitmq-wo-b4ns-direct.log" nerdctl run --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$SERVER_IP --producers 2 --consumers 2 --time 60 > $LOG_NAME nerdctl rm -f rabbitmq-server ) echo "===== Benchmark: rabbitmq client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f rabbitmq-server set -ex nerdctl run -d --name rabbitmq-server -p 5673:5672 $RABBITMQ_IMAGE sleep 10 SERVER_IP=$(nerdctl exec rabbitmq-server hostname -i) LOG_NAME="rabbitmq-wo-b4ns-host.log" nerdctl run --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$HOST_IP:5673 --producers 2 --consumers 2 --time 60 > $LOG_NAME nerdctl rm -f rabbitmq-server ) echo "===== Benchmark: rabbitmq client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e nerdctl rm -f rabbitmq-server systemctl --user stop run-bypass4netnsd systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d --name rabbitmq-server -p 5673:5672 $RABBITMQ_IMAGE sleep 10 LOG_NAME="rabbitmq-w-b4ns.log" nerdctl run --annotation nerdctl/bypass4netns=true --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$HOST_IP:5673 --producers 2 --consumers 2 --time 60 > $LOG_NAME nerdctl rm -f rabbitmq-server systemctl --user stop run-bypass4netnsd systemctl --user reset-failed ) echo "===== Benchmark: rabbitmq client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f rabbitmq-server systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d --name rabbitmq-server -p 5673:5672 $RABBITMQ_IMAGE sleep 10 SERVER_IP=$(nerdctl exec rabbitmq-server hostname -i) nerdctl run --annotation nerdctl/bypass4netns=true --name rabbitmq-client --rm $PERF_IMAGE --uri amqp://$SERVER_IP --producers 2 --consumers 2 --time 60 nerdctl rm -f rabbitmq-server systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed )bypass4netns-0.4.1/benchmark/rabbitmq/rabbitmq_multinode.sh000077500000000000000000000106251460472144300241350ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f rabbitmq-server NAME="test" exec_lxc nerdctl rm -f rabbitmq-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" RABBITMQ_VERSION=3.12.10 RABBITMQ_IMAGE="rabbitmq:$RABBITMQ_VERSION" PERF_VERSION="2.20.0" PERF_IMAGE="pivotalrabbitmq/perf-test:$PERF_VERSION" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $RABBITMQ_IMAGE NAME="test" exec_lxc sudo nerdctl pull --quiet $PERF_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $RABBITMQ_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $PERF_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: rabbitmq rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name rabbitmq-server -d $RABBITMQ_IMAGE" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh rabbitmq-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name rabbitmq-client -d --entrypoint '' $PERF_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh rabbitmq-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="rabbitmq-multinode-rootful.log" NAME="test2" exec_lxc sudo nerdctl exec rabbitmq-client java -jar /perf_test/perf-test.jar --uri amqp://$TEST1_VXLAN_ADDR --producers 2 --consumers 2 --time 60 > $LOG_NAME NAME="test" exec_lxc sudo nerdctl rm -f rabbitmq-server NAME="test2" exec_lxc sudo nerdctl rm -f rabbitmq-client ) echo "===== Benchmark: rabbitmq client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name rabbitmq-server -d $RABBITMQ_IMAGE" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh rabbitmq-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name rabbitmq-client -d --entrypoint '' $PERF_IMAGE /bin/sh -c 'sleep infinity'" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh rabbitmq-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR sleep 5 LOG_NAME="rabbitmq-multinode-wo-b4ns.log" NAME="test2" exec_lxc nerdctl exec rabbitmq-client java -jar /perf_test/perf-test.jar --uri amqp://$TEST1_VXLAN_ADDR --producers 2 --consumers 2 --time 60 > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f rabbitmq-server NAME="test2" exec_lxc nerdctl rm -f rabbitmq-client ) echo "===== Benchmark: rabbitmq client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -p 5673:5672 --name rabbitmq-server -d $RABBITMQ_IMAGE" NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true --name rabbitmq-client -d --entrypoint '' $PERF_IMAGE /bin/sh -c 'sleep infinity'" sleep 5 SERVER_IP=$(NAME="test" exec_lxc nerdctl exec rabbitmq-server hostname -i) LOG_NAME="rabbitmq-multinode-w-b4ns.log" NAME="test2" exec_lxc nerdctl exec rabbitmq-client java -jar /perf_test/perf-test.jar --uri amqp://$SERVER_IP --producers 2 --consumers 2 --time 60 > $LOG_NAME NAME="test" exec_lxc nerdctl rm -f rabbitmq-server NAME="test2" exec_lxc nerdctl rm -f rabbitmq-client ) bypass4netns-0.4.1/benchmark/rabbitmq/rabbitmq_plot.py000066400000000000000000000023771460472144300231330ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import csv import sys def load_data(filename): data = {} with open(filename) as f: line = f.readline() while line: line = line.strip() if "sending rate avg" in line: data["Sending"] = int(line.split(" ")[5]) if "receiving rate avg" in line: data["Receiving"] = int(line.split(" ")[5]) line = f.readline() return data BAR_WIDTH=0.25 labels=['Sending', 'Receiving'] data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH datas = [] for i in range(0, data_num): filename = sys.argv[1+i] data = load_data(filename) datas.append(data) fig = plt.figure() ax1 = fig.add_subplot() ax1.set_ylabel("messages / second") for i in range(0, data_num): filename = sys.argv[1+i] ax1.bar([BAR_WIDTH*i, factor + BAR_WIDTH*i], [datas[i][labels[0]], datas[i][labels[0]]], align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) h1, l1 = ax1.get_legend_handles_labels() ax1.legend(h1, l1, loc="upper left") plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/redis/000077500000000000000000000000001460472144300172165ustar00rootroot00000000000000bypass4netns-0.4.1/benchmark/redis/redis.sh000077500000000000000000000101771460472144300206710ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail cd $(dirname $0) REDIS_VERSION=7.2.3 REDIS_IMAGE="redis:${REDIS_VERSION}" source ~/.profile . ../param.bash sudo nerdctl pull --quiet $REDIS_IMAGE nerdctl pull --quiet $REDIS_IMAGE echo "===== Benchmark: redis rootful via NetNS =====" ( set +e sudo nerdctl rm -f redis-server sudo nerdctl rm -f redis-client set -ex sudo nerdctl run -d --name redis-server "${REDIS_IMAGE}" sudo nerdctl run -d --name redis-client "${REDIS_IMAGE}" sleep infinity SERVER_IP=$(sudo nerdctl exec redis-server hostname -i) sudo nerdctl exec redis-client redis-benchmark -q -h $SERVER_IP --csv > redis-rootful-direct.log cat redis-rootful-direct.log sudo nerdctl rm -f redis-server sudo nerdctl rm -f redis-client ) echo "===== Benchmark: redis rootful via host =====" ( set +e sudo nerdctl rm -f redis-server sudo nerdctl rm -f redis-client set -ex sudo nerdctl run -d -p 6380:6379 --name redis-server "${REDIS_IMAGE}" sudo nerdctl run -d --name redis-client "${REDIS_IMAGE}" sleep infinity sudo nerdctl exec redis-client redis-benchmark -q -h $HOST_IP -p 6380 --csv > redis-rootful-host.log cat redis-rootful-host.log sudo nerdctl rm -f redis-server sudo nerdctl rm -f redis-client ) echo "===== Benchmark: redis client(w/o bypass4netns) server(w/o bypass4netns) via intermediate NetNS =====" ( set +e nerdctl rm -f redis-server nerdctl rm -f redis-client set -ex nerdctl run -d --name redis-server "${REDIS_IMAGE}" nerdctl run -d --name redis-client "${REDIS_IMAGE}" sleep infinity SERVER_IP=$(nerdctl exec redis-server hostname -i) nerdctl exec redis-client redis-benchmark -q -h $SERVER_IP --csv > redis-wo-b4ns-direct.log cat redis-wo-b4ns-direct.log nerdctl rm -f redis-server nerdctl rm -f redis-client ) echo "===== Benchmark: redis client(w/o bypass4netns) server(w/o bypass4netns) via host =====" ( set +e nerdctl rm -f redis-server nerdctl rm -f redis-client set -ex nerdctl run -d -p 6380:6379 --name redis-server "${REDIS_IMAGE}" nerdctl run -d --name redis-client "${REDIS_IMAGE}" sleep infinity nerdctl exec redis-client redis-benchmark -q -h $HOST_IP -p 6380 --csv > redis-wo-b4ns-host.log cat redis-wo-b4ns-host.log nerdctl rm -f redis-server nerdctl rm -f redis-client ) echo "===== Benchmark: redis client(w/ bypass4netns) server(w/ bypass4netns) via host =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f redis-server nerdctl rm -f redis-client systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd nerdctl run --annotation nerdctl/bypass4netns=true -d -p 6380:6379 --name redis-server $REDIS_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name redis-client $REDIS_IMAGE sleep infinity nerdctl exec redis-client redis-benchmark -q -h $HOST_IP -p 6380 --csv > redis-w-b4ns.log cat redis-w-b4ns.log nerdctl rm -f redis-server nerdctl rm -f redis-client systemctl --user stop run-bypass4netnsd ) echo "===== Benchmark: redis client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( set +e nerdctl rm -f redis-server nerdctl rm -f redis-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$HOST_IP:2379 --advertise-client-urls http://$HOST_IP:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP nerdctl run --annotation nerdctl/bypass4netns=true -d -p 6380:6379 --name redis-server $REDIS_IMAGE nerdctl run --annotation nerdctl/bypass4netns=true -d --name redis-client $REDIS_IMAGE sleep infinity SERVER_IP=$(nerdctl exec redis-server hostname -i) # without 'sleep 1', benchmark is not performed.(race condition?) nerdctl exec redis-client /bin/sh -c "sleep 1 && redis-benchmark -q -h $SERVER_IP -p 6379 --csv" nerdctl rm -f redis-server nerdctl rm -f redis-client systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/benchmark/redis/redis_multinode.sh000077500000000000000000000074331460472144300227520ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ../../test/util.sh set +e NAME="test" exec_lxc sudo nerdctl rm -f redis-server NAME="test" exec_lxc nerdctl rm -f redis-server sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" REDIS_VERSION=7.2.3 REDIS_IMAGE="redis:${REDIS_VERSION}" set -eux -o pipefail NAME="test" exec_lxc sudo nerdctl pull --quiet $REDIS_IMAGE NAME="test" exec_lxc nerdctl pull --quiet $REDIS_IMAGE sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: redis rootful with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name redis-server -d $REDIS_IMAGE" NAME="test" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh redis-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && sudo nerdctl run -p 4789:4789/udp --privileged --name redis-client -d $REDIS_IMAGE sleep infinity" NAME="test2" exec_lxc sudo /home/ubuntu/bypass4netns/test/setup_vxlan.sh redis-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR NAME="test2" exec_lxc sudo nerdctl exec redis-client redis-benchmark -q -h $TEST1_VXLAN_ADDR --csv > redis-multinode-rootful.log NAME="test" exec_lxc sudo nerdctl rm -f redis-server NAME="test2" exec_lxc sudo nerdctl rm -f redis-client ) echo "===== Benchmark: redis client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name redis-server -d $REDIS_IMAGE" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh redis-server $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name redis-client -d $REDIS_IMAGE sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh redis-client $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR NAME="test2" exec_lxc nerdctl exec redis-client redis-benchmark -q -h $TEST1_VXLAN_ADDR --csv > redis-multinode-wo-b4ns.log NAME="test" exec_lxc nerdctl rm -f redis-server NAME="test2" exec_lxc nerdctl rm -f redis-client ) echo "===== Benchmark: redis client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d -p 6380:6379 --name redis-server $REDIS_IMAGE" SERVER_IP=$(NAME="test" exec_lxc nerdctl exec redis-server hostname -i) NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name redis-client $REDIS_IMAGE sleep infinity" NAME="test2" exec_lxc nerdctl exec redis-client /bin/sh -c "sleep 1 && redis-benchmark -q -h $SERVER_IP --csv" > redis-multinode-w-b4ns.log NAME="test" exec_lxc nerdctl rm -f redis-server NAME="test2" exec_lxc nerdctl rm -f redis-client ) bypass4netns-0.4.1/benchmark/redis/redis_plot.py000066400000000000000000000030101460472144300217260ustar00rootroot00000000000000import matplotlib.pyplot as plt import numpy as np import csv import sys def load_data(filename): data = {} with open(filename) as f: reader = csv.reader(f) next(reader, None) for row in reader: data[row[0]] = float(row[1]) return data BAR_WIDTH=0.25 labels_for_data=['PING_INLINE', 'PING_MBULK', 'SET', 'GET', 'INCR', 'LPUSH', 'RPUSH', 'LPOP', 'RPOP', 'SADD', 'HSET', 'SPOP', 'ZADD', 'ZPOPMIN', 'LPUSH (needed to benchmark LRANGE)', 'LRANGE_100 (first 100 elements)', 'LRANGE_300 (first 300 elements)', 'LRANGE_500 (first 500 elements)', 'LRANGE_600 (first 600 elements)', 'MSET (10 keys)', 'XADD'] labels=['PING\n_INLINE', 'PING\n_MBULK', 'SET', 'GET', 'INCR', 'LPUSH', 'RPUSH', 'LPOP', 'RPOP', 'SADD', 'HSET', 'SPOP', 'ZADD', 'ZPOPMIN', 'LPUSH', 'LRANGE\n_100', 'LRANGE\n_300', 'LRANGE\n_500', 'LRANGE\n_600', 'MSET\n(10 keys)', 'XADD'] plt.rcParams["figure.figsize"] = (20,4) plt.ylabel("Request / seconds") data_num = len(sys.argv)-2 factor = (data_num+1) * BAR_WIDTH for i in range(0, data_num): filename = sys.argv[1+i] data_csv = load_data(filename) value = [] for l in labels_for_data: value.append(data_csv[l]) plt.bar([x*factor+(BAR_WIDTH*i) for x in range(0, len(labels))], value, align="edge", edgecolor="black", linewidth=1, width=BAR_WIDTH, label=filename) plt.legend() plt.xlim(0, (len(labels)-1)*factor+BAR_WIDTH*data_num) plt.xticks([x*factor+BAR_WIDTH*data_num/2 for x in range(0, len(labels))], labels) plt.savefig(sys.argv[1+data_num]) bypass4netns-0.4.1/benchmark/run_bench.sh000077500000000000000000000005421460472144300204130ustar00rootroot00000000000000#!/bin/bash set -e cd $(dirname $0) BENCHMARKS=(iperf3 block redis memcached etcd rabbitmq mysql postgres) for BENCH in ${BENCHMARKS[@]}; do pushd $BENCH ./${BENCH}.sh python3 ${BENCH}_plot.py $BENCH-rootful-direct.log $BENCH-rootful-host.log $BENCH-wo-b4ns-direct.log $BENCH-wo-b4ns-host.log $BENCH-w-b4ns.log ../$BENCH.png popd donebypass4netns-0.4.1/cmd/000077500000000000000000000000001460472144300147215ustar00rootroot00000000000000bypass4netns-0.4.1/cmd/bypass4netns/000077500000000000000000000000001460472144300173565ustar00rootroot00000000000000bypass4netns-0.4.1/cmd/bypass4netns/main.go000066400000000000000000000200751460472144300206350ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "net" "os" "os/signal" "path/filepath" "strconv" "strings" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/nsagent" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/tracer" "github.com/rootless-containers/bypass4netns/pkg/oci" pkgversion "github.com/rootless-containers/bypass4netns/pkg/version" seccomp "github.com/seccomp/libseccomp-golang" "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" "golang.org/x/sys/unix" ) var ( socketFile string comSocketFile string pidFile string logFilePath string multinodeEtcdAddress string multinodeHostAddress string readyFd int exitFd int ) func main() { unix.Umask(0o077) // https://github.com/golang/go/issues/11822#issuecomment-123850227 xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") if xdgRuntimeDir == "" { panic("$XDG_RUNTIME_DIR needs to be set") } flag.StringVar(&socketFile, "socket", filepath.Join(xdgRuntimeDir, oci.SocketName), "Socket file") flag.StringVar(&comSocketFile, "com-socket", filepath.Join(xdgRuntimeDir, "bypass4netnsd-com.sock"), "Socket file for communication with bypass4netns") flag.StringVar(&pidFile, "pid-file", "", "Pid file") flag.StringVar(&logFilePath, "log-file", "", "Output logs to file") flag.StringVar(&multinodeEtcdAddress, "multinode-etcd-address", "", "Etcd address for multinode communication") flag.StringVar(&multinodeHostAddress, "multinode-host-address", "", "Host address for multinode communication") flag.IntVar(&readyFd, "ready-fd", -1, "File descriptor to notify when ready") flag.IntVar(&exitFd, "exit-fd", -1, "File descriptor for terminating bypass4netns") ignoredSubnets := flag.StringSlice("ignore", []string{"127.0.0.0/8"}, "Subnets to ignore in bypass4netns. Can be also set to \"auto\".") fowardPorts := flag.StringArrayP("publish", "p", []string{}, "Publish a container's port(s) to the host") debug := flag.Bool("debug", false, "Enable debug mode") version := flag.Bool("version", false, "Show version") help := flag.Bool("help", false, "Show help") nsagentFlag := flag.Bool("nsagent", false, "(An internal flag. Do not use manually.)") // TODO: hide tracerAgentFlag := flag.Bool("tracer-agent", false, "(An internal flag. Do not use manually.)") // TODO: hide memNSEnterPid := flag.Int("mem-nsenter-pid", -1, "(An internal flag. Do not use manually.)") // TODO: hide handleC2cEnable := flag.Bool("handle-c2c-connections", false, "Handle connections between containers") tracerEnable := flag.Bool("tracer", false, "Enable connection tracer") multinodeEnable := flag.Bool("multinode", false, "Enable multinode communication") ignoreBind := flag.Bool("ignore-bind", false, "Disable bypassing bind") // Parse arguments flag.Parse() if flag.NArg() > 0 { flag.PrintDefaults() logrus.Fatal("Invalid command") } if *debug { logrus.Info("Debug mode enabled") logrus.SetLevel(logrus.DebugLevel) } else { logrus.SetLevel(logrus.InfoLevel) } if *version { fmt.Printf("bypass4netns version %s\n", strings.TrimPrefix(pkgversion.Version, "v")) major, minor, micro := seccomp.GetLibraryVersion() fmt.Printf("libseccomp: %d.%d.%d\n", major, minor, micro) os.Exit(0) } if *help { flag.Usage() os.Exit(0) } if *memNSEnterPid > 0 { logrus.SetOutput(os.Stdout) if err := bypass4netns.OpenMemWithNSEnterAgent(uint32(*memNSEnterPid)); err != nil { logrus.Fatal(err) } os.Exit(0) } if logFilePath != "" { logFile, err := os.Create(logFilePath) if err != nil { logrus.Fatalf("Cannnot write log file %s : %v", logFilePath, err) } defer logFile.Close() logrus.SetOutput(io.MultiWriter(os.Stderr, logFile)) logrus.Infof("LogFilePath: %s", logFilePath) } if *nsagentFlag { if err := nsagent.Main(); err != nil { logrus.Fatal(err) } os.Exit(0) } if *tracerAgentFlag { if err := tracer.Main(); err != nil { logrus.Fatal(err) } os.Exit(0) } if *handleC2cEnable { if comSocketFile == "" { logrus.Fatal("--com-socket is not specified") } } if *tracerEnable { if !*handleC2cEnable { logrus.Fatal("--handle-c2c-connections is not enabled") } } if *multinodeEnable { if multinodeEtcdAddress == "" { logrus.Fatal("--multinode-etcd-address is not specified") } if multinodeHostAddress == "" { logrus.Fatal("--multinode-host-address is not specified") } logrus.WithFields(logrus.Fields{"etcdAddress": multinodeEtcdAddress, "hostAddress": multinodeHostAddress}).Infof("Multinode communication is enabled.") } if err := os.Remove(socketFile); err != nil && !errors.Is(err, os.ErrNotExist) { logrus.Fatalf("Cannot cleanup socket file: %v", err) } if pidFile != "" { pid := fmt.Sprintf("%d", os.Getpid()) if err := os.WriteFile(pidFile, []byte(pid), 0o644); err != nil { logrus.Fatalf("Cannot write pid file: %v", err) } logrus.Infof("PidFilePath: %s", pidFile) } logrus.Infof("SocketPath: %s", socketFile) handler := bypass4netns.NewHandler(socketFile, comSocketFile, strings.Replace(logFilePath, ".log", "-tracer.log", -1), *ignoreBind) subnets := []net.IPNet{} var subnetsAuto bool for _, subnetStr := range *ignoredSubnets { switch subnetStr { case "auto": if subnetsAuto { logrus.Warn("--ignore=\"auto\" appeared multiple times") } subnetsAuto = true logrus.Info("Enabling auto-update for --ignore") default: _, subnet, err := net.ParseCIDR(subnetStr) if err != nil { logrus.Fatalf("%s is not CIDR format", subnetStr) } subnets = append(subnets, *subnet) logrus.Infof("%s is added to ignore", subnet) } } handler.SetIgnoredSubnets(subnets, subnetsAuto) for _, forwardPortStr := range *fowardPorts { ports := strings.Split(forwardPortStr, ":") if len(ports) != 2 { logrus.Fatalf("invalid publish port format: '%s'", forwardPortStr) } hostPort, err := strconv.Atoi(ports[0]) if err != nil { logrus.Fatalf("not interger %s in '%s'", ports[0], forwardPortStr) } childPort, err := strconv.Atoi(ports[1]) if err != nil { logrus.Fatalf("not interger %s in '%s'", ports[1], forwardPortStr) } portMap := bypass4netns.ForwardPortMapping{ HostPort: hostPort, ChildPort: childPort, } err = handler.SetForwardingPort(portMap) if err != nil { logrus.Fatalf("failed to set fowardind port '%s' : %s", forwardPortStr, err) } logrus.Infof("fowarding port %s (host=%d container=%d) is added", forwardPortStr, hostPort, childPort) } if readyFd >= 0 { err := handler.SetReadyFd(readyFd) if err != nil { logrus.Fatalf("failed to set readyFd: %s", err) } } if exitFd >= 0 { exitFile := os.NewFile(uintptr(exitFd), "exit-fd") if exitFile == nil { logrus.Fatalf("invalid exit-fd %d", exitFd) } defer exitFile.Close() go func() { if _, err := io.ReadAll(exitFile); err != nil { logrus.Fatalf("Failed to wait for exit-fd %d to be closed: %v", exitFd, err) } pid := os.Getpid() logrus.Infof("The exit-fd was closed, sending SIGTERM to the process itself (PID %d)", pid) if err := unix.Kill(pid, unix.SIGTERM); err != nil { logrus.Fatalf("Failed to kill(%d, SIGTERM)", pid) } }() } go func() { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, unix.SIGTERM, unix.SIGINT) // SIGHUP is propagated to nsagents for reloading sig := <-sigCh logrus.Infof("Received signal %v, exiting...", sig) logrus.Infof("Removing socket %q", socketFile) if err := os.RemoveAll(socketFile); err != nil { logrus.Warnf("Failed to remove socket %q", socketFile) } if pidFile != "" { logrus.Infof("Removing pid file %q", pidFile) if err := os.RemoveAll(pidFile); err != nil { logrus.Warnf("Failed to remove pid file %q", pidFile) } } // The log file is not removed here os.Exit(0) }() c2cConfig := &bypass4netns.C2CConnectionHandleConfig{ Enable: *handleC2cEnable, TracerEnable: *tracerEnable, } multinode := &bypass4netns.MultinodeConfig{ Enable: *multinodeEnable, EtcdAddress: multinodeEtcdAddress, HostAddress: multinodeHostAddress, } handler.StartHandle(c2cConfig, multinode) } bypass4netns-0.4.1/cmd/bypass4netnsd/000077500000000000000000000000001460472144300175225ustar00rootroot00000000000000bypass4netns-0.4.1/cmd/bypass4netnsd/main.go000066400000000000000000000140151460472144300207760ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "net" "net/http" "os" "path/filepath" "strings" "github.com/gorilla/mux" "github.com/rootless-containers/bypass4netns/pkg/api/com" "github.com/rootless-containers/bypass4netns/pkg/api/daemon/router" "github.com/rootless-containers/bypass4netns/pkg/bypass4netnsd" pkgversion "github.com/rootless-containers/bypass4netns/pkg/version" "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" "golang.org/x/sys/unix" ) var ( socketFile string comSocketFile string // socket for channel with bypass4netns pidFile string logFilePath string b4nnPath string multinodeEtcdAddress string multinodeHostAddress string ) func main() { unix.Umask(0o077) // https://github.com/golang/go/issues/11822#issuecomment-123850227 xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") if xdgRuntimeDir == "" { logrus.Fatalf("$XDG_RUNTIME_DIR needs to be set") } exePath, err := os.Executable() if err != nil { logrus.Fatalf("failed to get myself executable path: %s", err) } defaultB4nnPath := filepath.Join(filepath.Dir(exePath), "bypass4netns") flag.StringVar(&socketFile, "socket", filepath.Join(xdgRuntimeDir, "bypass4netnsd.sock"), "Socket file") flag.StringVar(&comSocketFile, "com-socket", filepath.Join(xdgRuntimeDir, "bypass4netnsd-com.sock"), "Socket file for communication with bypass4netns") flag.StringVar(&pidFile, "pid-file", "", "Pid file") flag.StringVar(&logFilePath, "log-file", "", "Output logs to file") flag.StringVar(&b4nnPath, "b4nn-executable", defaultB4nnPath, "Path to bypass4netns executable") flag.StringVar(&multinodeEtcdAddress, "multinode-etcd-address", "", "Etcd address for multinode communication") flag.StringVar(&multinodeHostAddress, "multinode-host-address", "", "Host address for multinode communication") tracerEnable := flag.Bool("tracer", false, "Enable connection tracer") handleC2cEnable := flag.Bool("handle-c2c-connections", false, "Handle connections between containers") multinodeEnable := flag.Bool("multinode", false, "Enable multinode communication") debug := flag.Bool("debug", false, "Enable debug mode") version := flag.Bool("version", false, "Show version") help := flag.Bool("help", false, "Show help") // Parse arguments flag.Parse() if flag.NArg() > 0 { flag.PrintDefaults() logrus.Fatal("Invalid command") } if *debug { logrus.Info("Debug mode enabled") logrus.SetLevel(logrus.DebugLevel) } else { logrus.SetLevel(logrus.InfoLevel) } if *version { fmt.Printf("bypass4netnsd version %s\n", strings.TrimPrefix(pkgversion.Version, "v")) os.Exit(0) } if *help { flag.Usage() os.Exit(0) } if err := os.Remove(socketFile); err != nil && !errors.Is(err, os.ErrNotExist) { logrus.Fatalf("Cannot cleanup socket file: %v", err) } logrus.Infof("SocketPath: %s", socketFile) if err := os.Remove(comSocketFile); err != nil && !errors.Is(err, os.ErrNotExist) { logrus.Fatalf("Cannot cleanup communication socket file: %v", err) } logrus.Infof("CommunicationSocketPath: %s", comSocketFile) if pidFile != "" { pid := fmt.Sprintf("%d", os.Getpid()) if err := os.WriteFile(pidFile, []byte(pid), 0o644); err != nil { logrus.Fatalf("Cannot write pid file: %v", err) } logrus.Infof("PidFilePath: %s", pidFile) } if logFilePath != "" { logFile, err := os.Create(logFilePath) if err != nil { logrus.Fatalf("Cannnot write log file %s : %v", logFilePath, err) } defer logFile.Close() logrus.SetOutput(io.MultiWriter(os.Stderr, logFile)) logrus.Infof("LogFilePath %s", logFilePath) } if _, err = os.Stat(b4nnPath); err != nil { logrus.Fatalf("bypass4netns executable not found %s", b4nnPath) } logrus.Infof("bypass4netns executable path: %s", b4nnPath) b4nsdDriver := bypass4netnsd.NewDriver(b4nnPath, comSocketFile) if *handleC2cEnable && *multinodeEnable { logrus.Fatal("--handle-c2c-connections and multinode cannot be enabled at the sametime") } if *handleC2cEnable { logrus.Info("Handling connections between containers") b4nsdDriver.HandleC2CEnable = *handleC2cEnable } if *tracerEnable { if !*handleC2cEnable { logrus.Fatal("--handle-c2c-connections is not enabled") } logrus.Info("Connection tracer is enabled") b4nsdDriver.TracerEnable = *tracerEnable } if *multinodeEnable { if multinodeEtcdAddress == "" { logrus.Fatal("--multinode-etcd-address is not specified") } if multinodeHostAddress == "" { logrus.Fatal("--multinode-host-address is not specified") } b4nsdDriver.MultinodeEnable = *multinodeEnable b4nsdDriver.MultinodeEtcdAddress = multinodeEtcdAddress b4nsdDriver.MultinodeHostAddress = multinodeHostAddress logrus.WithFields(logrus.Fields{"etcdAddress": multinodeEtcdAddress, "hostAddress": multinodeHostAddress}).Info("Multinode communication is enabled.") } waitChan := make(chan bool) go func() { err = listenServeNerdctlAPI(socketFile, &router.Backend{ BypassDriver: b4nsdDriver, }) if err != nil { logrus.Fatalf("failed to serve nerdctl API: %q", err) } waitChan <- true }() go func() { err = listenServeBypass4netnsAPI(comSocketFile, &com.Backend{ BypassDriver: b4nsdDriver, }) if err != nil { logrus.Fatalf("failed to serve bypass4netns: %q", err) } waitChan <- true }() <-waitChan logrus.Fatalf("process exited") } func listenServeNerdctlAPI(socketPath string, backend *router.Backend) error { r := mux.NewRouter() router.AddRoutes(r, backend) srv := &http.Server{Handler: r} err := os.RemoveAll(socketPath) if err != nil { return err } l, err := net.Listen("unix", socketPath) if err != nil { return err } logrus.Infof("Starting nerdctl API to serve on %s", socketPath) return srv.Serve(l) } func listenServeBypass4netnsAPI(sockPath string, backend *com.Backend) error { r := mux.NewRouter() com.AddRoutes(r, backend) srv := &http.Server{Handler: r} err := os.RemoveAll(sockPath) if err != nil { return err } l, err := net.Listen("unix", sockPath) if err != nil { return err } logrus.Infof("Starting bypass4netns API to serve on %s", sockPath) return srv.Serve(l) } bypass4netns-0.4.1/cmd/bypass4netnsd/main_test.go000066400000000000000000000073671460472144300220510ustar00rootroot00000000000000package main import ( "context" "net" "os" "path/filepath" "syscall" "testing" "github.com/rootless-containers/bypass4netns/pkg/api" "github.com/rootless-containers/bypass4netns/pkg/api/com" "github.com/rootless-containers/bypass4netns/pkg/api/daemon/client" "github.com/stretchr/testify/assert" ) // Start bypass4netnsd before testing func TestNerdctlAPI(t *testing.T) { xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") if xdgRuntimeDir == "" { panic("$XDG_RUNTIME_DIR needs to be set") } client, err := client.New(filepath.Join(xdgRuntimeDir, "bypass4netnsd.sock")) if err != nil { t.Fatalf("failed client.New %s", err) } bm := client.BypassManager() specs := api.BypassSpec{ ID: "1234567890", } status, err := bm.StartBypass(context.TODO(), specs) assert.Equal(t, nil, err) statuses, err := bm.ListBypass(context.TODO()) assert.Equal(t, nil, err) assert.Equal(t, 1, len(statuses)) newStatus := statuses[0] assert.Equal(t, status.ID, newStatus.ID) assert.NotEqual(t, 0, newStatus.Pid) assert.Equal(t, true, isProcessRunning(newStatus.Pid)) err = bm.StopBypass(context.TODO(), specs.ID) assert.Equal(t, nil, err) assert.Equal(t, false, isProcessRunning(newStatus.Pid)) statuses, err = bm.ListBypass(context.TODO()) assert.Equal(t, nil, err) assert.Equal(t, 0, len(statuses)) } func isProcessRunning(pid int) bool { proc, err := os.FindProcess(pid) if err != nil { return false } // check the process is alive or not err = proc.Signal(syscall.Signal(0)) return err == nil } func TestBypass4netnsAPI(t *testing.T) { xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") if xdgRuntimeDir == "" { panic("$XDG_RUNTIME_DIR needs to be set") } client, err := com.NewComClient(filepath.Join(xdgRuntimeDir, "bypass4netnsd-com.sock")) if err != nil { t.Fatalf("failed to create ComClient %q", err) } mac, err := net.ParseMAC("ea:1e:d5:cd:e2:ea") assert.Equal(t, nil, err) ip, ipNet, err := net.ParseCIDR("10.4.0.53/24") assert.Equal(t, nil, err) ipNet.IP = ip cid := "c70ae35d2aeb4c98c5ef9eb4" containerIf := com.ContainerInterfaces{ ContainerID: cid, Interfaces: []com.Interface{ { Name: "eth0", HWAddr: mac, Addresses: []net.IPNet{*ipNet}, IsLoopback: false, }, }, ForwardingPorts: map[int]int{ 5201: 5202, }, } err = client.Ping(context.TODO()) assert.Equal(t, nil, err) ifs, err := client.ListInterfaces(context.TODO()) assert.Equal(t, nil, err) assert.Equal(t, 0, len(ifs)) // this should be error _, err = client.GetInterface(context.TODO(), containerIf.ContainerID) assert.NotEqual(t, nil, err) // Registering interface postedIfs, err := client.PostInterface(context.TODO(), &containerIf) assert.Equal(t, nil, err) assert.Equal(t, postedIfs.ContainerID, containerIf.ContainerID) assert.Equal(t, postedIfs.Interfaces[0].HWAddr, containerIf.Interfaces[0].HWAddr) ifs2, err := client.ListInterfaces(context.TODO()) assert.Equal(t, nil, err) assert.Equal(t, 1, len(ifs2)) assert.Equal(t, ifs2[cid].ContainerID, containerIf.ContainerID) assert.Equal(t, ifs2[cid].Interfaces[0].HWAddr, containerIf.Interfaces[0].HWAddr) assert.Equal(t, ifs2[cid].ForwardingPorts[5201], 5202) ifs3, err := client.GetInterface(context.TODO(), containerIf.ContainerID) assert.Equal(t, nil, err) assert.Equal(t, ifs3.ContainerID, containerIf.ContainerID) assert.Equal(t, ifs3.Interfaces[0].HWAddr, containerIf.Interfaces[0].HWAddr) assert.Equal(t, ifs3.ForwardingPorts[5201], 5202) // Removing interface err = client.DeleteInterface(context.TODO(), containerIf.ContainerID) assert.Equal(t, nil, err) ifs4, err := client.ListInterfaces(context.TODO()) assert.Equal(t, nil, err) assert.Equal(t, 0, len(ifs4)) _, err = client.GetInterface(context.TODO(), containerIf.ContainerID) assert.NotEqual(t, nil, err) } bypass4netns-0.4.1/go.mod000066400000000000000000000025721460472144300152720ustar00rootroot00000000000000module github.com/rootless-containers/bypass4netns go 1.21 require ( github.com/gorilla/mux v1.8.1 github.com/opencontainers/runtime-spec v1.2.0 github.com/seccomp/libseccomp-golang v0.10.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 go.etcd.io/etcd/client/v3 v3.5.13 golang.org/x/sys v0.19.0 ) require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) bypass4netns-0.4.1/go.sum000066400000000000000000000224221460472144300153130ustar00rootroot00000000000000github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 h1:X6ps8XHfpQjw8dUStzlMi2ybiKQ2Fmdw7UM+TinwvyM= github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810/go.mod h1:dF0BBJ2YrV1+2eAIyEI+KeSidgA6HqoIP1u5XTlMq/o= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= bypass4netns-0.4.1/pkg/000077500000000000000000000000001460472144300147375ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/000077500000000000000000000000001460472144300155105ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/api.go000066400000000000000000000014131460472144300166070ustar00rootroot00000000000000package api type BypassStatus struct { ID string `json:"id"` Pid int `json:"pid"` Spec BypassSpec `json:"spec"` } type BypassSpec struct { ID string `json:"id"` SocketPath string `json:"socketPath"` PidFilePath string `json:"pidFilePath"` LogFilePath string `json:"logFilePath"` PortMapping []PortSpec `json:"portMapping"` IgnoreSubnets []string `json:"ignoreSubnets"` // CIDR or "auto" IgnoreBind bool `json:"ignoreBind"` } type PortSpec struct { Protos []string `json:"protos"` ParentIP string `json:"parentIP"` ParentPort int `json:"parentPort"` ChildIP string `json:"childIP"` ChildPort int `json:"childPort"` } type ErrorJSON struct { Message string `json:"message"` } bypass4netns-0.4.1/pkg/api/com/000077500000000000000000000000001460472144300162665ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/com/api.go000066400000000000000000000007421460472144300173710ustar00rootroot00000000000000package com import ( "net" ) type ContainerInterfaces struct { ContainerID string `json:"containerID"` Interfaces []Interface `json:"interfaces"` // key is "container-side" port, value is host-side port ForwardingPorts map[int]int `json:"forwardingPorts"` } type Interface struct { Name string `json:"name"` HWAddr net.HardwareAddr `json:"hwAddr"` Addresses []net.IPNet `json:"addresses"` IsLoopback bool `json:"isLoopback"` } bypass4netns-0.4.1/pkg/api/com/client.go000066400000000000000000000127731460472144300201050ustar00rootroot00000000000000// This code is copied from https://github.com/rootless-containers/rootlesskit/blob/master/pkg/api/client/client.go v0.14.6 // The code is licensed under Apache-2.0 package com import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "os" "github.com/rootless-containers/bypass4netns/pkg/api" ) type ComClient struct { client *http.Client version string dummyHost string } func NewComClient(socketPath string) (*ComClient, error) { if _, err := os.Stat(socketPath); err != nil { return nil, err } hc := &http.Client{ Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, "unix", socketPath) }, }, } return &ComClient{ client: hc, version: "v1", dummyHost: "bypass4netnsd-com", }, nil } func readAtMost(r io.Reader, maxBytes int) ([]byte, error) { lr := &io.LimitedReader{ R: r, N: int64(maxBytes), } b, err := io.ReadAll(lr) if err != nil { return b, err } if lr.N == 0 { return b, fmt.Errorf("expected at most %d bytes, got more", maxBytes) } return b, nil } // HTTPStatusErrorBodyMaxLength specifies the maximum length of HTTPStatusError.Body const HTTPStatusErrorBodyMaxLength = 64 * 1024 // HTTPStatusError is created from non-2XX HTTP response type HTTPStatusError struct { // StatusCode is non-2XX status code StatusCode int // Body is at most HTTPStatusErrorBodyMaxLength Body string } // Error implements error. // If e.Body is a marshalled string of api.ErrorJSON, Error returns ErrorJSON.Message . // Otherwise Error returns a human-readable string that contains e.StatusCode and e.Body. func (e *HTTPStatusError) Error() string { if e.Body != "" && len(e.Body) < HTTPStatusErrorBodyMaxLength { var ej api.ErrorJSON if json.Unmarshal([]byte(e.Body), &ej) == nil { return ej.Message } } return fmt.Sprintf("unexpected HTTP status %s, body=%q", http.StatusText(e.StatusCode), e.Body) } func successful(resp *http.Response) error { if resp == nil { return errors.New("nil response") } if resp.StatusCode/100 != 2 { b, _ := readAtMost(resp.Body, HTTPStatusErrorBodyMaxLength) return &HTTPStatusError{ StatusCode: resp.StatusCode, Body: string(b), } } return nil } func (c *ComClient) Ping(ctx context.Context) error { m, err := json.Marshal("ping") if err != nil { return err } u := fmt.Sprintf("http://%s/%s/ping", c.dummyHost, c.version) req, err := http.NewRequest("GET", u, bytes.NewReader(m)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { return err } defer resp.Body.Close() if err := successful(resp); err != nil { return err } dec := json.NewDecoder(resp.Body) var pong string if err := dec.Decode(&pong); err != nil { return err } if pong != "pong" { return fmt.Errorf("unexpected response expected=%q actual=%q", "pong", pong) } return nil } func (c *ComClient) ListInterfaces(ctx context.Context) (map[string]ContainerInterfaces, error) { u := fmt.Sprintf("http://%s/%s/interfaces", c.dummyHost, c.version) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if err := successful(resp); err != nil { return nil, err } dec := json.NewDecoder(resp.Body) var containerIfs map[string]ContainerInterfaces if err := dec.Decode(&containerIfs); err != nil { return nil, err } return containerIfs, nil } func (c *ComClient) GetInterface(ctx context.Context, id string) (*ContainerInterfaces, error) { u := fmt.Sprintf("http://%s/%s/interface/%s", c.dummyHost, c.version, id) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if err := successful(resp); err != nil { return nil, err } dec := json.NewDecoder(resp.Body) var containerIfs ContainerInterfaces if err := dec.Decode(&containerIfs); err != nil { return nil, err } return &containerIfs, nil } func (c *ComClient) PostInterface(ctx context.Context, ifs *ContainerInterfaces) (*ContainerInterfaces, error) { m, err := json.Marshal(ifs) if err != nil { return nil, err } u := fmt.Sprintf("http://%s/%s/interface/%s", c.dummyHost, c.version, ifs.ContainerID) req, err := http.NewRequest("POST", u, bytes.NewReader(m)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if err := successful(resp); err != nil { return nil, err } dec := json.NewDecoder(resp.Body) var containerIfs ContainerInterfaces if err := dec.Decode(&containerIfs); err != nil { return nil, err } return &containerIfs, nil } func (c *ComClient) DeleteInterface(ctx context.Context, id string) error { u := fmt.Sprintf("http://%s/%s/interface/%s", c.dummyHost, c.version, id) req, err := http.NewRequest("DELETE", u, nil) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := c.client.Do(req) if err != nil { return err } defer resp.Body.Close() if err := successful(resp); err != nil { return err } return nil } bypass4netns-0.4.1/pkg/api/com/router.go000066400000000000000000000063101460472144300201350ustar00rootroot00000000000000package com import ( "encoding/json" "errors" "net/http" "github.com/gorilla/mux" "github.com/rootless-containers/bypass4netns/pkg/api" ) type Backend struct { BypassDriver BypassDriver } type BypassDriver interface { ListInterfaces() map[string]ContainerInterfaces GetInterface(id string) *ContainerInterfaces PostInterface(id string, containerIfs *ContainerInterfaces) DeleteInterface(id string) } func AddRoutes(r *mux.Router, b *Backend) { v1 := r.PathPrefix("/v1").Subrouter() _ = v1 v1.Path("/ping").Methods("GET").HandlerFunc(b.ping) v1.Path("/interfaces").Methods("GET").HandlerFunc(b.listInterfaces) v1.Path("/interface/{id}").Methods("GET").HandlerFunc(b.getInterface) v1.Path("/interface/{id}").Methods("POST").HandlerFunc(b.postInterface) v1.Path("/interface/{id}").Methods("DELETE").HandlerFunc(b.deleteInterface) } func (b *Backend) onError(w http.ResponseWriter, r *http.Request, err error, ec int) { w.WriteHeader(ec) w.Header().Set("Content-Type", "application/json") // it is safe to return the err to the client, because the client is reliable e := api.ErrorJSON{ Message: err.Error(), } _ = json.NewEncoder(w).Encode(e) } func (b *Backend) ping(w http.ResponseWriter, r *http.Request) { m, err := json.Marshal("pong") if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(m) } func (b *Backend) listInterfaces(w http.ResponseWriter, r *http.Request) { ifs := b.BypassDriver.ListInterfaces() m, err := json.Marshal(ifs) if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(m) } func (b *Backend) getInterface(w http.ResponseWriter, r *http.Request) { id, ok := mux.Vars(r)["id"] if !ok { b.onError(w, r, errors.New("id not specified"), http.StatusBadRequest) return } ifs := b.BypassDriver.GetInterface(id) if ifs == nil { b.onError(w, r, errors.New("not found"), http.StatusNotFound) return } m, err := json.Marshal(ifs) if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(m) } func (b *Backend) postInterface(w http.ResponseWriter, r *http.Request) { id, ok := mux.Vars(r)["id"] if !ok { b.onError(w, r, errors.New("id not specified"), http.StatusBadRequest) return } decoder := json.NewDecoder(r.Body) var containerIfs ContainerInterfaces if err := decoder.Decode(&containerIfs); err != nil { b.onError(w, r, err, http.StatusBadRequest) return } b.BypassDriver.PostInterface(id, &containerIfs) ifs := b.BypassDriver.GetInterface(id) m, err := json.Marshal(ifs) if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(m) } func (b *Backend) deleteInterface(w http.ResponseWriter, r *http.Request) { id, ok := mux.Vars(r)["id"] if !ok { b.onError(w, r, errors.New("id not specified"), http.StatusBadRequest) return } b.BypassDriver.DeleteInterface(id) } bypass4netns-0.4.1/pkg/api/daemon/000077500000000000000000000000001460472144300167535ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/daemon/client/000077500000000000000000000000001460472144300202315ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/daemon/client/client.go000066400000000000000000000110171460472144300220360ustar00rootroot00000000000000// This code is copied from https://github.com/rootless-containers/rootlesskit/blob/master/pkg/api/client/client.go v0.14.6 // The code is licensed under Apache-2.0 package client import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "os" "github.com/rootless-containers/bypass4netns/pkg/api" ) type Client interface { HTTPClient() *http.Client BypassManager() *BypassManager } // New creates a client. // socketPath is a path to the UNIX socket, without unix:// prefix. func New(socketPath string) (Client, error) { if _, err := os.Stat(socketPath); err != nil { return nil, err } hc := &http.Client{ Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, "unix", socketPath) }, }, } return NewWithHTTPClient(hc), nil } func NewWithHTTPClient(hc *http.Client) Client { return &client{ Client: hc, version: "v1", dummyHost: "bypass4netnsd", } } type client struct { *http.Client // version is always "v1" // TODO(AkihiroSuda): negotiate the version version string dummyHost string } func (c *client) HTTPClient() *http.Client { return c.Client } func (c *client) BypassManager() *BypassManager { return &BypassManager{ client: c, } } func readAtMost(r io.Reader, maxBytes int) ([]byte, error) { lr := &io.LimitedReader{ R: r, N: int64(maxBytes), } b, err := io.ReadAll(lr) if err != nil { return b, err } if lr.N == 0 { return b, fmt.Errorf("expected at most %d bytes, got more", maxBytes) } return b, nil } // HTTPStatusErrorBodyMaxLength specifies the maximum length of HTTPStatusError.Body const HTTPStatusErrorBodyMaxLength = 64 * 1024 // HTTPStatusError is created from non-2XX HTTP response type HTTPStatusError struct { // StatusCode is non-2XX status code StatusCode int // Body is at most HTTPStatusErrorBodyMaxLength Body string } // Error implements error. // If e.Body is a marshalled string of api.ErrorJSON, Error returns ErrorJSON.Message . // Otherwise Error returns a human-readable string that contains e.StatusCode and e.Body. func (e *HTTPStatusError) Error() string { if e.Body != "" && len(e.Body) < HTTPStatusErrorBodyMaxLength { var ej api.ErrorJSON if json.Unmarshal([]byte(e.Body), &ej) == nil { return ej.Message } } return fmt.Sprintf("unexpected HTTP status %s, body=%q", http.StatusText(e.StatusCode), e.Body) } func successful(resp *http.Response) error { if resp == nil { return errors.New("nil response") } if resp.StatusCode/100 != 2 { b, _ := readAtMost(resp.Body, HTTPStatusErrorBodyMaxLength) return &HTTPStatusError{ StatusCode: resp.StatusCode, Body: string(b), } } return nil } type BypassManager struct { *client } func (bm *BypassManager) StartBypass(ctx context.Context, spec api.BypassSpec) (*api.BypassStatus, error) { m, err := json.Marshal(spec) if err != nil { return nil, err } u := fmt.Sprintf("http://%s/%s/bypass", bm.client.dummyHost, bm.client.version) req, err := http.NewRequest("POST", u, bytes.NewReader(m)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req = req.WithContext(ctx) resp, err := bm.client.HTTPClient().Do(req) if err != nil { return nil, err } defer resp.Body.Close() if err := successful(resp); err != nil { return nil, err } dec := json.NewDecoder(resp.Body) var status api.BypassStatus if err := dec.Decode(&status); err != nil { return nil, err } return &status, nil } func (bm *BypassManager) ListBypass(ctx context.Context) ([]api.BypassStatus, error) { u := fmt.Sprintf("http://%s/%s/bypass", bm.client.dummyHost, bm.client.version) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } req = req.WithContext(ctx) resp, err := bm.client.HTTPClient().Do(req) if err != nil { return nil, err } defer resp.Body.Close() if err := successful(resp); err != nil { return nil, err } var statuses []api.BypassStatus dec := json.NewDecoder(resp.Body) if err := dec.Decode(&statuses); err != nil { return nil, err } return statuses, nil } func (bm *BypassManager) StopBypass(ctx context.Context, id string) error { u := fmt.Sprintf("http://%s/%s/bypass/%s", bm.client.dummyHost, bm.client.version, id) req, err := http.NewRequest("DELETE", u, nil) if err != nil { return err } req = req.WithContext(ctx) resp, err := bm.client.HTTPClient().Do(req) if err != nil { return err } defer resp.Body.Close() if err := successful(resp); err != nil { return err } return nil } bypass4netns-0.4.1/pkg/api/daemon/openapi.yaml000066400000000000000000000050161460472144300212740ustar00rootroot00000000000000# This code is copied from https://github.com/rootless-containers/rootlesskit/blob/master/pkg/api/openapi.yaml v0.14.6 # The code is licensed under Apache-2.0 openapi: 3.0.3 info: title: bypass4netnsd API version: 0.0.1 servers: - url: 'http://bypass4netnsd/v1' paths: /bypass: get: responses: '200': description: An array of BypassStatus content: application/json: schema: $ref: '#/components/schemas/BypassStatuses' post: requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/BypassSpec' responses: '201': description: BypassStatus content: application/json: schema: $ref: '#/components/schemas/BypassStatus' /bypass/{id}: delete: parameters: - name: id in: path required: true schema: type: string responses: '200': description: Null response components: schemas: Proto: type: string description: "protocol for listening. Corresponds to Go's net.Listen." enum: - tcp - tcp4 - tcp6 - udp - udp4 - udp6 - sctp - sctp4 - sctp6 BypassStatuses: type: array items: $ref: '#/components/schemas/BypassStatus' BypassStatus: required: - id - pid properties: id: type: string pid: type: integer spec: $ref: '#/components/schemas/BypassSpec' BypassSpec: required: - id properties: id: type: string socketPath: type: string pidFilePath: type: string logFilePath: type: string portMapping: type: array items: $ref: '#/components/schemas/PortSpec' ignoreSubnets: type: array items: type: string PortSpec: properties: protos: type: array items: $ref: '#/components/schemas/Proto' parentIP: type: string parentPort: type: integer format: int32 minimum: 1 maximum: 65535 childIP: type: string childPort: type: integer format: int32 minimum: 1 maximum: 65535bypass4netns-0.4.1/pkg/api/daemon/router/000077500000000000000000000000001460472144300202735ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/api/daemon/router/router.go000066400000000000000000000042571460472144300221520ustar00rootroot00000000000000package router import ( "encoding/json" "errors" "net/http" "github.com/gorilla/mux" "github.com/rootless-containers/bypass4netns/pkg/api" ) type Backend struct { BypassDriver BypassDriver } type BypassDriver interface { ListBypass() []api.BypassStatus StartBypass(*api.BypassSpec) (*api.BypassStatus, error) StopBypass(id string) error } func (b *Backend) onError(w http.ResponseWriter, r *http.Request, err error, ec int) { w.WriteHeader(ec) w.Header().Set("Content-Type", "application/json") // it is safe to return the err to the client, because the client is reliable e := api.ErrorJSON{ Message: err.Error(), } _ = json.NewEncoder(w).Encode(e) } func (b *Backend) GetBypasses(w http.ResponseWriter, r *http.Request) { bs := b.BypassDriver.ListBypass() m, err := json.Marshal(bs) if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write(m) } func (b *Backend) PostBypass(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) var bSpec api.BypassSpec if err := decoder.Decode(&bSpec); err != nil { b.onError(w, r, err, http.StatusBadRequest) return } bypassStatus, err := b.BypassDriver.StartBypass(&bSpec) if err != nil { b.onError(w, r, err, http.StatusBadRequest) return } m, err := json.Marshal(bypassStatus) if err != nil { b.onError(w, r, err, http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) _, _ = w.Write(m) } func (b *Backend) DeleteBypass(w http.ResponseWriter, r *http.Request) { id, ok := mux.Vars(r)["id"] if !ok { b.onError(w, r, errors.New("id not specified"), http.StatusBadRequest) return } if err := b.BypassDriver.StopBypass(id); err != nil { b.onError(w, r, err, http.StatusBadRequest) return } w.WriteHeader(http.StatusOK) } func AddRoutes(r *mux.Router, b *Backend) { v1 := r.PathPrefix("/v1").Subrouter() v1.Path("/bypass").Methods("GET").HandlerFunc(b.GetBypasses) v1.Path("/bypass").Methods("POST").HandlerFunc(b.PostBypass) v1.Path("/bypass/{id}").Methods("DELETE").HandlerFunc(b.DeleteBypass) } bypass4netns-0.4.1/pkg/bypass4netns/000077500000000000000000000000001460472144300173745ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/bypass4netns.go000066400000000000000000000717621460472144300223750ustar00rootroot00000000000000package bypass4netns // This code is copied from 'runc(https://github.com/opencontainers/runc/blob/v1.1.0/contrib/cmd/seccompagent/seccompagent.go)' // The code is licensed under Apache-2.0 License import ( "bytes" gocontext "context" "encoding/json" "errors" "fmt" "net" "os" "os/exec" "strconv" "strings" "syscall" "time" "github.com/opencontainers/runtime-spec/specs-go" "github.com/rootless-containers/bypass4netns/pkg/api/com" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/iproute2" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/nonbypassable" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/tracer" "github.com/rootless-containers/bypass4netns/pkg/util" libseccomp "github.com/seccomp/libseccomp-golang" "github.com/sirupsen/logrus" clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/sys/unix" ) const ETCD_MULTINODE_PREFIX = "bypass4netns/multinode/" func closeStateFds(recvFds []int) { for i := range recvFds { unix.Close(i) } } // parseStateFds returns the seccomp-fd and closes the rest of the fds in recvFds. // In case of error, no fd is closed. // StateFds is assumed to be formatted as specs.ContainerProcessState.Fds and // recvFds the corresponding list of received fds in the same SCM_RIGHT message. func parseStateFds(stateFds []string, recvFds []int) (uintptr, error) { // Let's find the index in stateFds of the seccomp-fd. idx := -1 err := false for i, name := range stateFds { if name == specs.SeccompFdName && idx == -1 { idx = i continue } // We found the seccompFdName twice. Error out! if name == specs.SeccompFdName && idx != -1 { err = true } } if idx == -1 || err { return 0, errors.New("seccomp fd not found or malformed containerProcessState.Fds") } if idx >= len(recvFds) || idx < 0 { return 0, errors.New("seccomp fd index out of range") } fd := uintptr(recvFds[idx]) for i := range recvFds { if i == idx { continue } unix.Close(recvFds[i]) } return fd, nil } // readProcMem read data from memory of specified pid process at the spcified offset. func (h *notifHandler) readProcMem(pid int, offset uint64, len uint64) ([]byte, error) { buffer := make([]byte, len) // PATH_MAX memfd, err := h.openMem(pid) if err != nil { return nil, err } size, err := unix.Pread(memfd, buffer, int64(offset)) if err != nil { return nil, err } return buffer[:size], nil } // writeProcMem writes data to memory of specified pid process at the specified offset. func (h *notifHandler) writeProcMem(pid int, offset uint64, buf []byte) error { memfd, err := h.openMem(pid) if err != nil { return err } size, err := unix.Pwrite(memfd, buf, int64(offset)) if err != nil { return err } if len(buf) != size { return fmt.Errorf("data is not written successfully. expected size=%d actual size=%d", len(buf), size) } return nil } func (h *notifHandler) openMem(pid int) (int, error) { if memfd, ok := h.memfds[pid]; ok { return memfd, nil } memfd, err := unix.Open(fmt.Sprintf("/proc/%d/mem", pid), unix.O_RDWR, 0o777) if err != nil { logrus.WithField("pid", pid).Warn("failed to open mem due to permission error. retrying with agent.") newMemfd, err := openMemWithNSEnter(pid) if err != nil { return 0, fmt.Errorf("failed to open mem with agent (pid=%d)", pid) } logrus.WithField("pid", pid).Info("succeeded to open mem with agent. continue to process") memfd = newMemfd } h.memfds[pid] = memfd return memfd, nil } func openMemWithNSEnter(pid int) (int, error) { fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) if err != nil { return 0, err } // configure timeout timeout := &syscall.Timeval{ Sec: 0, Usec: 500 * 1000, } err = syscall.SetsockoptTimeval(fds[0], syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, timeout) if err != nil { return 0, fmt.Errorf("failed to set receive timeout") } err = syscall.SetsockoptTimeval(fds[1], syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, timeout) if err != nil { return 0, fmt.Errorf("failed to set send timeout") } fd1File := os.NewFile(uintptr(fds[0]), "") defer fd1File.Close() fd1Conn, err := net.FileConn(fd1File) if err != nil { return 0, err } _ = fd1Conn selfExe, err := os.Executable() if err != nil { return 0, err } nsenter, err := exec.LookPath("nsenter") if err != nil { return 0, err } nsenterFlags := []string{ "-t", strconv.Itoa(int(pid)), "-F", } selfPid := os.Getpid() ok, err := util.SameUserNS(int(pid), selfPid) if err != nil { return 0, fmt.Errorf("failed to check sameUserNS(%d, %d)", pid, selfPid) } if !ok { nsenterFlags = append(nsenterFlags, "-U", "--preserve-credentials") } nsenterFlags = append(nsenterFlags, "--", selfExe, fmt.Sprintf("--mem-nsenter-pid=%d", pid)) cmd := exec.CommandContext(gocontext.TODO(), nsenter, nsenterFlags...) cmd.ExtraFiles = []*os.File{os.NewFile(uintptr(fds[1]), "")} stdout := bytes.Buffer{} cmd.Stdout = &stdout err = cmd.Start() if err != nil { return 0, fmt.Errorf("failed to exec mem open agent %q", err) } memfd, recvMsgs, err := util.RecvMsg(fd1Conn) if err != nil { logrus.Infof("stdout=%q", stdout.String()) return 0, fmt.Errorf("failed to receive message") } logrus.Debugf("recvMsgs=%s", string(recvMsgs)) err = cmd.Wait() if err != nil { return 0, err } return memfd, nil } func OpenMemWithNSEnterAgent(pid uint32) error { // fd 3 should be passed socket pair fdFile := os.NewFile(uintptr(3), "") defer fdFile.Close() fdConn, err := net.FileConn(fdFile) if err != nil { logrus.WithError(err).Fatal("failed to open conn") } memPath := fmt.Sprintf("/proc/%d/mem", pid) memfd, err := unix.Open(memPath, unix.O_RDWR, 0o777) if err != nil { logrus.WithError(err).Fatalf("failed to open %s", memPath) } err = util.SendMsg(fdConn, memfd, []byte(fmt.Sprintf("opened %s", memPath))) if err != nil { logrus.WithError(err).Fatal("failed to send message") } return nil } func handleNewMessage(sockfd int) (uintptr, *specs.ContainerProcessState, error) { const maxNameLen = 4096 stateBuf := make([]byte, maxNameLen) oobSpace := unix.CmsgSpace(4) oob := make([]byte, oobSpace) n, oobn, _, _, err := unix.Recvmsg(sockfd, stateBuf, oob, 0) if err != nil { return 0, nil, err } if n >= maxNameLen || oobn != oobSpace { return 0, nil, fmt.Errorf("recvfd: incorrect number of bytes read (n=%d oobn=%d)", n, oobn) } // Truncate. stateBuf = stateBuf[:n] oob = oob[:oobn] scms, err := unix.ParseSocketControlMessage(oob) if err != nil { return 0, nil, err } if len(scms) != 1 { return 0, nil, fmt.Errorf("recvfd: number of SCMs is not 1: %d", len(scms)) } scm := scms[0] fds, err := unix.ParseUnixRights(&scm) if err != nil { return 0, nil, err } containerProcessState := &specs.ContainerProcessState{} err = json.Unmarshal(stateBuf, containerProcessState) if err != nil { closeStateFds(fds) return 0, nil, fmt.Errorf("cannot parse OCI state: %w", err) } fd, err := parseStateFds(containerProcessState.Fds, fds) if err != nil { closeStateFds(fds) return 0, nil, err } return fd, containerProcessState, nil } type context struct { notifFd libseccomp.ScmpFd req *libseccomp.ScmpNotifReq resp *libseccomp.ScmpNotifResp } func (h *notifHandler) getPidFdInfo(pid int) (*pidInfo, error) { // retrieve pidfd from cache if pidfd, ok := h.pidInfos[pid]; ok { return &pidfd, nil } targetPidfd, err := unix.PidfdOpen(int(pid), 0) if err == nil { info := pidInfo{ pidType: PROCESS, pidfd: targetPidfd, tgid: pid, // process's pid is equal to its tgid } h.pidInfos[pid] = info return &info, nil } // pid can be thread and pidfd_open fails with thread's pid. // retrieve process's pid (tgid) from /proc//status and retry to get pidfd with the tgid. logrus.Warnf("pidfd Open failed: pid=%d err=%q, this pid maybe thread and retrying with tgid", pid, err) st, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) if err != nil { return nil, fmt.Errorf("failed to read %d's status err=%q", pid, err) } nextTgid := -1 for _, s := range strings.Split(string(st), "\n") { if strings.Contains(s, "Tgid") { tgids := strings.Split(s, "\t") if len(tgids) < 2 { return nil, fmt.Errorf("unexpected /proc/%d/status len=%q status=%q", pid, len(tgids), string(st)) } tgid, err := strconv.Atoi(tgids[1]) if err != nil { return nil, fmt.Errorf("unexpected /proc/%d/status err=%q status=%q", pid, err, string(st)) } nextTgid = tgid } if nextTgid > 0 { break } } if nextTgid < 0 { logrus.Errorf("cannot get Tgid from /proc/%d/status status=%q", pid, string(st)) } targetPidfd, err = unix.PidfdOpen(nextTgid, 0) if err != nil { return nil, fmt.Errorf("pidfd Open failed with Tgid: pid=%d %s", nextTgid, err) } logrus.Infof("successfully got pidfd for pid=%d tgid=%d", pid, nextTgid) info := pidInfo{ pidType: THREAD, pidfd: targetPidfd, tgid: nextTgid, } h.pidInfos[pid] = info return &info, nil } // getFdInProcess get the file descriptor in other process func (h *notifHandler) getFdInProcess(pid, targetFd int) (int, error) { targetPidfd, err := h.getPidFdInfo(pid) if err != nil { return 0, fmt.Errorf("pidfd Open failed: %s", err) } fd, err := unix.PidfdGetfd(targetPidfd.pidfd, targetFd, 0) if err != nil { return 0, fmt.Errorf("pidfd GetFd failed: %s", err) } return fd, nil } // getSocketArgs retrieves socket(2) arguemnts from fd. // return values are (sock_domain, sock_type, sock_protocol, error) func getSocketArgs(sockfd int) (int, int, int, error) { logrus.Debugf("got sockfd=%v", sockfd) sock_domain, err := syscall.GetsockoptInt(sockfd, syscall.SOL_SOCKET, syscall.SO_DOMAIN) if err != nil { return 0, 0, 0, fmt.Errorf("getsockopt(SO_DOMAIN) failed: %s", err) } sock_type, err := syscall.GetsockoptInt(sockfd, syscall.SOL_SOCKET, syscall.SO_TYPE) if err != nil { return 0, 0, 0, fmt.Errorf("getsockopt(SO_TYPE) failed: %s", err) } sock_protocol, err := syscall.GetsockoptInt(sockfd, syscall.SOL_SOCKET, syscall.SO_PROTOCOL) if err != nil { return 0, 0, 0, fmt.Errorf("getsockopt(SO_PROTOCOL) failed: %s", err) } return sock_domain, sock_type, sock_protocol, nil } func (h *notifHandler) readSockaddrFromProcess(pid int, offset uint64, addrlen uint64) (*sockaddr, error) { buf, err := h.readProcMem(pid, offset, addrlen) if err != nil { return nil, fmt.Errorf("failed readProcMem pid %v offset 0x%x: %s", pid, offset, err) } return newSockaddr(buf) } func (h *notifHandler) registerSocket(pid int, sockfd int, syscallName string) (*socketStatus, error) { logger := logrus.WithFields(logrus.Fields{"pid": pid, "sockfd": sockfd, "syscall": syscallName}) proc, ok := h.processes[pid] if !ok { proc = newProcessStatus() h.processes[pid] = proc logger.Debug("process is registered") } sock, ok := proc.sockets[sockfd] if ok { logger.Warn("socket is already registered") return sock, nil } // If the pid is thread, its process can have corresponding socket procInfo, ok := h.pidInfos[int(pid)] if ok && procInfo.pidType == THREAD { return nil, fmt.Errorf("unexpected procInfo") } sockFdHost, err := h.getFdInProcess(int(pid), sockfd) if err != nil { return nil, err } defer syscall.Close(sockFdHost) sockDomain, sockType, sockProtocol, err := getSocketArgs(sockFdHost) sock = newSocketStatus(pid, sockfd, sockDomain, sockType, sockProtocol, h.ignoreBind) if err != nil { // non-socket fd is not bypassable sock.state = NotBypassable logger.Debugf("failed to get socket args err=%q", err) } else { if sockDomain != syscall.AF_INET && sockDomain != syscall.AF_INET6 { // non IP sockets are not handled. sock.state = NotBypassable logger.Debugf("socket domain=0x%x", sockDomain) } else if sockType != syscall.SOCK_STREAM { // only accepting TCP socket sock.state = NotBypassable logger.Debugf("socket type=0x%x", sockType) } else { // only newly created socket is allowed. _, err := syscall.Getpeername(sockFdHost) if err == nil { logger.Infof("socket is already connected. socket is created via accept or forked") sock.state = NotBypassable } } } proc.sockets[sockfd] = sock if sock.state == NotBypassable { logger.Debugf("socket is registered (state=%s)", sock.state) } else { logger.Infof("socket is registered (state=%s)", sock.state) } return sock, nil } func (h *notifHandler) getSocket(pid int, sockfd int) *socketStatus { proc, ok := h.processes[pid] if !ok { return nil } sock := proc.sockets[sockfd] return sock } func (h *notifHandler) removeSocket(pid int, sockfd int) { defer logrus.WithFields(logrus.Fields{"pid": pid, "sockfd": sockfd}).Debugf("socket is removed") proc, ok := h.processes[pid] if !ok { return } delete(proc.sockets, sockfd) } // handleReq handles seccomp notif requests and configures responses. func (h *notifHandler) handleReq(ctx *context) { syscallName, err := ctx.req.Data.Syscall.GetName() if err != nil { logrus.Errorf("Error decoding syscall %v(): %s", ctx.req.Data.Syscall, err) // TODO: error handle return } logrus.Tracef("Received syscall %q, pid %v, arch %q, args %+v", syscallName, ctx.req.Pid, ctx.req.Data.Arch, ctx.req.Data.Args) ctx.resp.Flags |= SeccompUserNotifFlagContinue // ensure pid is registered in notifHandler.pidInfos pidInfo, err := h.getPidFdInfo(int(ctx.req.Pid)) if err != nil { logrus.Errorf("failed to get pidfd err=%q", err) return } // threads shares file descriptors in the same process space. // so use tgid as pid to process socket file descriptors pid := pidInfo.tgid if pidInfo.pidType == THREAD { logrus.Debugf("pid %d is thread. use process's tgid %d as pid", ctx.req.Pid, pid) } // cleanup sockets when the process exit. if syscallName == "_exit" || syscallName == "exit_group" { if pidInfo, ok := h.pidInfos[int(ctx.req.Pid)]; ok { syscall.Close(int(pidInfo.pidfd)) delete(h.pidInfos, int(ctx.req.Pid)) } if pidInfo.pidType == THREAD { logrus.WithFields(logrus.Fields{"pid": ctx.req.Pid, "tgid": pid}).Infof("thread is removed") } if pidInfo.pidType == PROCESS { delete(h.processes, pid) if memfd, ok := h.memfds[pid]; ok { syscall.Close(memfd) delete(h.memfds, pid) } logrus.WithFields(logrus.Fields{"pid": pid}).Infof("process is removed") } return } sockfd := int(ctx.req.Data.Args[0]) // remove socket when closed if syscallName == "close" { h.removeSocket(pid, sockfd) return } sock := h.getSocket(pid, sockfd) if sock == nil { sock, err = h.registerSocket(pid, sockfd, syscallName) if err != nil { logrus.Errorf("failed to register socket pid %d sockfd %d: %s", pid, sockfd, err) return } } switch sock.state { case NotBypassable: // sometimes close(2) is not called for the fd. // To handle such condition, re-register fd when connect is called for not bypassable fd. if syscallName == "connect" { h.removeSocket(pid, sockfd) sock, err = h.registerSocket(pid, sockfd, syscallName) if err != nil { logrus.Errorf("failed to re-register socket pid %d sockfd %d: %s", pid, sockfd, err) return } } if sock.state != NotBypassed { return } // when sock.state == NotBypassed, continue case Bypassed: if syscallName == "getpeername" { sock.handleSysGetpeername(h, ctx) } return default: } switch syscallName { case "bind": sock.handleSysBind(pid, h, ctx) case "connect": sock.handleSysConnect(h, ctx) case "setsockopt": sock.handleSysSetsockopt(pid, h, ctx) case "fcntl": sock.handleSysFcntl(ctx) case "getpeername": // already handled default: logrus.Errorf("Unknown syscall %q", syscallName) // TODO: error handle return } } // notifHandler handles seccomp notifications and response to them. func (h *notifHandler) handle() { defer unix.Close(int(h.fd)) if h.nonBypassableAutoUpdate { go func() { if nbErr := h.nonBypassable.WatchNS(gocontext.TODO(), h.state.Pid); nbErr != nil { logrus.WithError(nbErr).Fatalf("failed to watch NS (PID=%d)", h.state.Pid) } }() } for { req, err := libseccomp.NotifReceive(h.fd) if err != nil { logrus.Errorf("Error in NotifReceive(): %s", err) continue } ctx := context{ notifFd: h.fd, req: req, resp: &libseccomp.ScmpNotifResp{ ID: req.ID, Error: 0, Val: 0, Flags: libseccomp.NotifRespFlagContinue, }, } // TOCTOU check if err := libseccomp.NotifIDValid(h.fd, req.ID); err != nil { logrus.Errorf("TOCTOU check failed: req.ID is no longer valid: %s", err) continue } h.handleReq(&ctx) if err = libseccomp.NotifRespond(h.fd, ctx.resp); err != nil { logrus.Errorf("Error in notification response: %s", err) continue } } } type ForwardPortMapping struct { HostPort int ChildPort int } type Handler struct { socketPath string comSocketPath string tracerAgentLogPath string ignoredSubnets []net.IPNet ignoredSubnetsAutoUpdate bool readyFd int // key is child port forwardingPorts map[int]ForwardPortMapping ignoreBind bool } // NewHandler creates new seccomp notif handler func NewHandler(socketPath, comSocketPath, tracerAgentLogPath string, ignoreBind bool) *Handler { handler := Handler{ socketPath: socketPath, comSocketPath: comSocketPath, tracerAgentLogPath: tracerAgentLogPath, ignoredSubnets: []net.IPNet{}, forwardingPorts: map[int]ForwardPortMapping{}, readyFd: -1, ignoreBind: ignoreBind, } return &handler } // SetIgnoreSubnets configures subnets to ignore in bypass4netns. func (h *Handler) SetIgnoredSubnets(subnets []net.IPNet, autoUpdate bool) { h.ignoredSubnets = subnets h.ignoredSubnetsAutoUpdate = autoUpdate } // SetForwardingPort checks and configures port forwarding func (h *Handler) SetForwardingPort(mapping ForwardPortMapping) error { for _, fwd := range h.forwardingPorts { if fwd.HostPort == mapping.HostPort { return fmt.Errorf("host port %d is already forwarded", fwd.HostPort) } if fwd.ChildPort == mapping.ChildPort { return fmt.Errorf("container port %d is already forwarded", fwd.ChildPort) } } h.forwardingPorts[mapping.ChildPort] = mapping return nil } // SetReadyFd configure ready notification file descriptor func (h *Handler) SetReadyFd(fd int) error { if fd < 0 { return fmt.Errorf("ready-fd must be a non-negative integer") } h.readyFd = fd return nil } type MultinodeConfig struct { Enable bool EtcdAddress string HostAddress string etcdClientConfig clientv3.Config etcdClient *clientv3.Client } type C2CConnectionHandleConfig struct { Enable bool TracerEnable bool } type notifHandler struct { fd libseccomp.ScmpFd state *specs.ContainerProcessState nonBypassable *nonbypassable.NonBypassable nonBypassableAutoUpdate bool // key is child port forwardingPorts map[int]ForwardPortMapping // key is pid processes map[int]*processStatus // key is destination address e.g. "192.168.1.1:1000" containerInterfaces map[string]containerInterface c2cConnections *C2CConnectionHandleConfig multinode *MultinodeConfig // cache /proc//mem's fd to reduce latency. key is pid, value is fd memfds map[int]int // cache pidfd to reduce latency. key is pid. pidInfos map[int]pidInfo ignoreBind bool } type containerInterface struct { containerID string hostPort int lastCheckedUnix int64 } type pidInfoPidType int const ( PROCESS pidInfoPidType = iota THREAD ) type pidInfo struct { pidType pidInfoPidType pidfd int tgid int } func (h *Handler) newNotifHandler(fd uintptr, state *specs.ContainerProcessState) *notifHandler { notifHandler := notifHandler{ fd: libseccomp.ScmpFd(fd), state: state, forwardingPorts: map[int]ForwardPortMapping{}, processes: map[int]*processStatus{}, memfds: map[int]int{}, pidInfos: map[int]pidInfo{}, ignoreBind: h.ignoreBind, } notifHandler.nonBypassable = nonbypassable.New(h.ignoredSubnets) notifHandler.nonBypassableAutoUpdate = h.ignoredSubnetsAutoUpdate // Deep copy of map for key, value := range h.forwardingPorts { notifHandler.forwardingPorts[key] = value } return ¬ifHandler } // StartHandle starts seccomp notif handler func (h *Handler) StartHandle(c2cConfig *C2CConnectionHandleConfig, multinodeConfig *MultinodeConfig) { logrus.Info("Waiting for seccomp file descriptors") l, err := net.Listen("unix", h.socketPath) if err != nil { logrus.Fatalf("Cannot listen: %v", err) } defer l.Close() if h.readyFd >= 0 { logrus.Infof("notify ready fd=%d", h.readyFd) _, err = syscall.Write(h.readyFd, []byte{1}) if err != nil { logrus.Fatalf("failed to notify fd=%d", h.readyFd) } syscall.Close(h.readyFd) } // prepare tracer agent var tracerAgent *tracer.Tracer = nil for { conn, err := l.Accept() logrus.Info("accept connection") if err != nil { logrus.Errorf("Cannot accept connection: %s", err) continue } socket, err := conn.(*net.UnixConn).File() conn.Close() if err != nil { logrus.Errorf("Cannot get socket: %v", err) continue } newFd, state, err := handleNewMessage(int(socket.Fd())) socket.Close() if err != nil { logrus.Errorf("Error receiving seccomp file descriptor: %v", err) continue } logrus.Infof("Received new seccomp fd: %v", newFd) notifHandler := h.newNotifHandler(newFd, state) notifHandler.c2cConnections = c2cConfig notifHandler.multinode = multinodeConfig if notifHandler.multinode.Enable { notifHandler.multinode.etcdClientConfig = clientv3.Config{ Endpoints: []string{notifHandler.multinode.EtcdAddress}, } notifHandler.multinode.etcdClient, err = clientv3.New(notifHandler.multinode.etcdClientConfig) if err != nil { logrus.WithError(err).Fatal("failed to create etcd client") } } // not to run multiple tracerAgent. // TODO: prepare only one tracerAgent in Handler if c2cConfig.TracerEnable && !multinodeConfig.Enable && tracerAgent == nil { tracerAgent = tracer.NewTracer(h.tracerAgentLogPath) err = tracerAgent.StartTracer(gocontext.TODO(), state.Pid) if err != nil { logrus.WithError(err).Fatalf("failed to start tracer") } fwdPorts := []int{} for _, v := range notifHandler.forwardingPorts { fwdPorts = append(fwdPorts, v.ChildPort) } err = tracerAgent.RegisterForwardPorts(fwdPorts) if err != nil { logrus.WithError(err).Fatalf("failed to register port") } logrus.WithField("fwdPorts", fwdPorts).Info("registered ports to tracer agent") // check tracer agent is ready for _, v := range fwdPorts { dst := fmt.Sprintf("127.0.0.1:%d", v) addr, err := tracerAgent.ConnectToAddress([]string{dst}) if err != nil { logrus.WithError(err).Warnf("failed to connect to %s", dst) continue } if len(addr) != 1 || addr[0] != dst { logrus.Fatalf("failed to connect to %s", dst) continue } logrus.Debugf("successfully connected to %s", dst) } logrus.Infof("tracer is ready") } else { logrus.Infof("tracer is disabled") } // TODO: these goroutines shoud be launched only once. ready := make(chan bool, 10) if notifHandler.multinode.Enable { go notifHandler.startBackgroundMultinodeTask(ready) } else if notifHandler.c2cConnections.Enable { go notifHandler.startBackgroundC2CConnectionHandleTask(ready, h.comSocketPath, tracerAgent) } else { ready <- true } // wait for background tasks becoming ready <-ready logrus.Info("background task is ready. start to handle") go notifHandler.handle() } } func (h *notifHandler) startBackgroundC2CConnectionHandleTask(ready chan bool, comSocketPath string, tracerAgent *tracer.Tracer) { initDone := false logrus.Info("Started bypass4netns background task") comClient, err := com.NewComClient(comSocketPath) if err != nil { logrus.Fatalf("failed to create ComClient: %q", err) } err = comClient.Ping(gocontext.TODO()) if err != nil { logrus.Fatalf("failed to connect to bypass4netnsd: %q", err) } logrus.Infof("Successfully connected to bypass4netnsd") ifLastUpdateUnix := int64(0) for { if ifLastUpdateUnix+10 < time.Now().Unix() { addrs, err := iproute2.GetAddressesInNetNS(gocontext.TODO(), h.state.Pid) if err != nil { logrus.WithError(err).Errorf("failed to get addresses") return } ifs, err := iproute2AddressesToComInterfaces(addrs) if err != nil { logrus.WithError(err).Errorf("failed to convert addresses") return } containerIfs := &com.ContainerInterfaces{ ContainerID: h.state.State.ID, Interfaces: ifs, ForwardingPorts: map[int]int{}, } for _, v := range h.forwardingPorts { containerIfs.ForwardingPorts[v.ChildPort] = v.HostPort } logrus.Debugf("Interfaces = %v", containerIfs) _, err = comClient.PostInterface(gocontext.TODO(), containerIfs) if err != nil { logrus.WithError(err).Errorf("failed to post interfaces") } else { logrus.Infof("successfully posted updated interfaces") ifLastUpdateUnix = time.Now().Unix() } } containerInterfaces, err := comClient.ListInterfaces(gocontext.TODO()) if err != nil { logrus.WithError(err).Warn("failed to list container interfaces") } containerIf := map[string]containerInterface{} for _, cont := range containerInterfaces { for contPort, hostPort := range cont.ForwardingPorts { for _, intf := range cont.Interfaces { if intf.IsLoopback { continue } for _, addr := range intf.Addresses { // ignore ipv6 address if addr.IP.To4() == nil { continue } dstAddr := fmt.Sprintf("%s:%d", addr.IP, contPort) contIf, ok := h.containerInterfaces[dstAddr] if ok && contIf.lastCheckedUnix+10 > time.Now().Unix() { containerIf[dstAddr] = contIf continue } if h.c2cConnections.TracerEnable { addrRes, err := tracerAgent.ConnectToAddress([]string{dstAddr}) if err != nil { logrus.WithError(err).Debugf("failed to connect to %s", dstAddr) continue } if len(addrRes) != 1 || addrRes[0] != dstAddr { logrus.Debugf("failed to connect to %s", dstAddr) continue } logrus.Debugf("successfully connected to %s", dstAddr) } containerIf[dstAddr] = containerInterface{ containerID: cont.ContainerID, hostPort: hostPort, lastCheckedUnix: time.Now().Unix(), } logrus.Infof("%s -> 127.0.0.1:%d is registered", dstAddr, hostPort) } } } } h.containerInterfaces = containerIf // once the interfaces are registered, it is ready to handle connections if !initDone { initDone = true ready <- true } time.Sleep(1 * time.Second) } } func iproute2AddressesToComInterfaces(addrs iproute2.Addresses) ([]com.Interface, error) { comIntfs := []com.Interface{} for _, intf := range addrs { comIntf := com.Interface{ Name: intf.IfName, Addresses: []net.IPNet{}, IsLoopback: intf.LinkType == "loopback", } hwAddr, err := net.ParseMAC(intf.Address) if err != nil { return nil, fmt.Errorf("failed to parse HWAddress: %w", err) } comIntf.HWAddr = hwAddr for _, addr := range intf.AddrInfos { ip, ipNet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", addr.Local, addr.PrefixLen)) if err != nil { return nil, fmt.Errorf("failed to parse addr_info: %w", err) } ipNet.IP = ip comIntf.Addresses = append(comIntf.Addresses, *ipNet) } comIntfs = append(comIntfs, comIntf) } return comIntfs, nil } func (h *notifHandler) startBackgroundMultinodeTask(ready chan bool) { initDone := false ifLastUpdateUnix := int64(0) for { if ifLastUpdateUnix+10 < time.Now().Unix() { ifs, err := iproute2.GetAddressesInNetNS(gocontext.TODO(), h.state.Pid) if err != nil { logrus.WithError(err).Errorf("failed to get addresses") return } for _, intf := range ifs { // ignore non-ethernet interface if intf.LinkType != "ether" { continue } for _, addr := range intf.AddrInfos { // ignore non-IPv4 address if addr.Family != "inet" { continue } for _, v := range h.forwardingPorts { containerAddr := fmt.Sprintf("%s:%d", addr.Local, v.ChildPort) hostAddr := fmt.Sprintf("%s:%d", h.multinode.HostAddress, v.HostPort) // Remove entries with timeout // TODO: Remove related entries when exiting. ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 2*time.Second) lease, err := h.multinode.etcdClient.Grant(ctx, 15) cancel() if err != nil { logrus.WithError(err).Errorf("failed to grant lease to register %s -> %s", containerAddr, hostAddr) continue } ctx, cancel = gocontext.WithTimeout(gocontext.Background(), 2*time.Second) _, err = h.multinode.etcdClient.Put(ctx, ETCD_MULTINODE_PREFIX+containerAddr, hostAddr, clientv3.WithLease(lease.ID)) cancel() if err != nil { logrus.WithError(err).Errorf("failed to register %s -> %s", containerAddr, hostAddr) } else { logrus.Infof("Registered %s -> %s", containerAddr, hostAddr) } } } } ifLastUpdateUnix = time.Now().Unix() // once the interfaces are registered, it is ready to handle connections if !initDone { initDone = true ready <- true } } time.Sleep(1 * time.Second) } } bypass4netns-0.4.1/pkg/bypass4netns/iproute2/000077500000000000000000000000001460472144300211455ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/iproute2/iproute2.go000066400000000000000000000042341460472144300232500ustar00rootroot00000000000000package iproute2 import ( "context" "encoding/json" "fmt" "os" "os/exec" "strconv" "github.com/rootless-containers/bypass4netns/pkg/util" "golang.org/x/sys/unix" ) type AddrInfo struct { Family string `json:"family"` Local string `json:"local"` PrefixLen int `json:"prefixlen"` Broadcast string `json:"broadcast"` Scope string `json:"scope"` Label string `json:"label"` ValidLifeTime int `json:"valid_life_time"` PreferredLifeTime int `json:"preferred_life_time"` } type Interface struct { IfIndex int `json:"ifindex"` IfName string `json:"ifname"` Flags []string `json:"flags"` Mtu int `json:"mtu"` Qdisc string `json:"noqueue"` Operstate string `json:"operstate"` Group string `json:"group"` TxQLen int `json:"txqlen"` LinkType string `json:"link_type"` Address string `json:"address"` Broadcast string `json:"broadcast"` AddrInfos []AddrInfo `json:"addr_info"` } type Addresses = []Interface func UnmarshalAddress(jsonAddrs []byte) (Addresses, error) { var addrs = Addresses{} err := json.Unmarshal(jsonAddrs, &addrs) if err != nil { return nil, err } return addrs, nil } func GetAddressesInNetNS(ctx context.Context, pid int) (Addresses, error) { nsenter, err := exec.LookPath("nsenter") if err != nil { return nil, err } nsenterFlags := []string{ "-t", strconv.Itoa(pid), "-F", "-n", } selfPid := os.Getpid() ok, err := util.SameUserNS(pid, selfPid) if err != nil { return nil, fmt.Errorf("failed to check sameUserNS(%d, %d)", pid, selfPid) } if !ok { nsenterFlags = append(nsenterFlags, "-U", "--preserve-credentials") } nsenterFlags = append(nsenterFlags, "--", "ip", "-j", "addr", "show") cmd := exec.CommandContext(ctx, nsenter, nsenterFlags...) cmd.SysProcAttr = &unix.SysProcAttr{ Pdeathsig: unix.SIGTERM, } stdout, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to start %v: %w", cmd.Args, err) } addrs, err := UnmarshalAddress(stdout) if err != nil { return nil, fmt.Errorf("failed to parse json: %w", err) } return addrs, nil } bypass4netns-0.4.1/pkg/bypass4netns/iproute2/iproute2_test.go000066400000000000000000000106231460472144300243060ustar00rootroot00000000000000package iproute2 import ( "fmt" "net" "testing" "github.com/stretchr/testify/assert" ) func TestUnmarshalAddress(t *testing.T) { testJson := ` [ { "ifindex":1, "ifname":"lo", "flags":[ "LOOPBACK", "UP", "LOWER_UP" ], "mtu":65536, "qdisc":"noqueue", "operstate":"UNKNOWN", "group":"default", "txqlen":1000, "link_type":"loopback", "address":"00:00:00:00:00:00", "broadcast":"00:00:00:00:00:00", "addr_info":[ { "family":"inet", "local":"127.0.0.1", "prefixlen":8, "scope":"host", "label":"lo", "valid_life_time":4294967295, "preferred_life_time":4294967295 }, { "family":"inet6", "local":"::1", "prefixlen":128, "scope":"host", "valid_life_time":4294967295, "preferred_life_time":4294967295 } ] }, { "ifindex":2, "ifname":"enp1s0", "flags":[ "BROADCAST", "MULTICAST", "UP", "LOWER_UP" ], "mtu":1500, "qdisc":"fq_codel", "operstate":"UP", "group":"default", "txqlen":1000, "link_type":"ether", "address":"52:54:00:c3:92:b6", "broadcast":"ff:ff:ff:ff:ff:ff", "addr_info":[ { "family":"inet", "local":"192.168.1.155", "prefixlen":24, "broadcast":"192.168.1.255", "scope":"global", "label":"enp1s0", "valid_life_time":4294967295, "preferred_life_time":4294967295 }, { "family":"inet6", "local":"fe80::5054:ff:fec3:92b6", "prefixlen":64, "scope":"link", "valid_life_time":4294967295, "preferred_life_time":4294967295 } ] }, { "ifindex":3, "ifname":"docker0", "flags":[ "NO-CARRIER", "BROADCAST", "MULTICAST", "UP" ], "mtu":1500, "qdisc":"noqueue", "operstate":"DOWN", "group":"default", "link_type":"ether", "address":"02:42:ab:c8:78:84", "broadcast":"ff:ff:ff:ff:ff:ff", "addr_info":[ { "family":"inet", "local":"172.17.0.1", "prefixlen":16, "broadcast":"172.17.255.255", "scope":"global", "label":"docker0", "valid_life_time":4294967295, "preferred_life_time":4294967295 } ] }, { "ifindex":61, "ifname":"lxdbr0", "flags":[ "BROADCAST", "MULTICAST", "UP", "LOWER_UP" ], "mtu":1500, "qdisc":"noqueue", "operstate":"UP", "group":"default", "txqlen":1000, "link_type":"ether", "address":"00:16:3e:4d:92:98", "broadcast":"ff:ff:ff:ff:ff:ff", "addr_info":[ { "family":"inet", "local":"192.168.6.1", "prefixlen":24, "scope":"global", "label":"lxdbr0", "valid_life_time":4294967295, "preferred_life_time":4294967295 } ] }, { "ifindex":71, "link_index":70, "ifname":"veth71db11e7", "flags":[ "BROADCAST", "MULTICAST", "UP", "LOWER_UP" ], "mtu":1500, "qdisc":"noqueue", "master":"lxdbr0", "operstate":"UP", "group":"default", "txqlen":1000, "link_type":"ether", "address":"da:83:f0:97:c7:14", "broadcast":"ff:ff:ff:ff:ff:ff", "link_netnsid":0, "addr_info":[ ] } ] ` addrs, err := UnmarshalAddress([]byte(testJson)) assert.Equal(t, nil, err) assert.Equal(t, 5, len(addrs)) intf := addrs[1] assert.Equal(t, "UP", intf.Operstate) assert.Equal(t, "ether", intf.LinkType) assert.Equal(t, 2, len(intf.AddrInfos)) addr := intf.AddrInfos[0] assert.Equal(t, "inet", addr.Family) assert.Equal(t, "192.168.1.155", addr.Local) addrIp, addrCidr, err := net.ParseCIDR(fmt.Sprintf("%s/%d", addr.Local, addr.PrefixLen)) assert.Equal(t, nil, err) addrCidr.IP = addrIp assert.Equal(t, "192.168.1.155/24", addrCidr.String()) addr2 := intf.AddrInfos[1] assert.Equal(t, "inet6", addr2.Family) assert.Equal(t, "fe80::5054:ff:fec3:92b6", addr2.Local) } bypass4netns-0.4.1/pkg/bypass4netns/nonbypassable/000077500000000000000000000000001460472144300222345ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/nonbypassable/nonbypassable.go000066400000000000000000000100331460472144300254200ustar00rootroot00000000000000package nonbypassable import ( "bufio" "context" "encoding/json" "errors" "fmt" "io" "net" "os" "os/exec" "os/signal" "strconv" "sync" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/nsagent/types" "github.com/rootless-containers/bypass4netns/pkg/util" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) func New(staticList []net.IPNet) *NonBypassable { x := &NonBypassable{ staticList: staticList, } return x } // NonBypassable maintains the list of the non-bypassable CIDRs, // such as 127.0.0.0/8 and CNI bridge CIDRs in the slirp's network namespace. type NonBypassable struct { staticList []net.IPNet dynamicList []net.IPNet mu sync.RWMutex } func (x *NonBypassable) Contains(ip net.IP) bool { x.mu.RLock() defer x.mu.RUnlock() for _, subnet := range append(x.staticList, x.dynamicList...) { if subnet.Contains(ip) { return true } } return false } //func (x *NonBypassable) IsInterfaceIPAddress(ip net.IP) bool { // x.mu.RLock() // defer x.mu.RUnlock() // for _, intf := range x.interfaces { // for _, intfIP := range intf.Addresses { // if intfIP.IP.Equal(ip) { // return true // } // } // } // // return false //} // //func (x *NonBypassable) GetInterfaces() []com.Interface { // x.mu.RLock() // defer x.mu.RUnlock() // ips := append([]com.Interface{}, x.interfaces...) // return ips //} // //func (x *NonBypassable) GetLastUpdateUnix() int64 { // x.mu.RLock() // defer x.mu.RUnlock() // return x.lastUpdateUnix //} // WatchNS watches the NS associated with the PID and updates the internal dynamic list on receiving SIGHUP. func (x *NonBypassable) WatchNS(ctx context.Context, pid int) error { selfExe, err := os.Executable() if err != nil { return err } nsenter, err := exec.LookPath("nsenter") if err != nil { return err } nsenterFlags := []string{ "-t", strconv.Itoa(pid), "-F", "-n", } selfPid := os.Getpid() ok, err := util.SameUserNS(pid, selfPid) if err != nil { return fmt.Errorf("failed to check sameUserNS(%d, %d)", pid, selfPid) } if !ok { nsenterFlags = append(nsenterFlags, "-U", "--preserve-credentials") } nsenterFlags = append(nsenterFlags, "--", selfExe, "--nsagent") cmd := exec.CommandContext(ctx, nsenter, nsenterFlags...) cmd.SysProcAttr = &unix.SysProcAttr{ Pdeathsig: unix.SIGTERM, } cmd.Stderr = os.Stderr r, w := io.Pipe() cmd.Stdout = w if err := cmd.Start(); err != nil { return fmt.Errorf("failed to start %v: %w", cmd.Args, err) } cmdPid := cmd.Process.Pid logrus.Infof("Dynamic non-bypassable list: started NSAgent (PID=%d, target PID=%d)", cmdPid, pid) go x.watchNS(r) // > It is allowed to call Notify multiple times with different channels and the same signals: // > each channel receives copies of incoming signals independently. // https://pkg.go.dev/os/signal#Notify sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, unix.SIGHUP) for sig := range sigCh { if uSig, ok := sig.(unix.Signal); ok { _ = unix.Kill(cmdPid, uSig) } } return nil } func (x *NonBypassable) watchNS(r io.Reader) { scanner := bufio.NewScanner(r) for scanner.Scan() { line := scanner.Text() var msg types.Message if err := json.Unmarshal([]byte(line), &msg); err != nil { logrus.WithError(err).Warnf("Dynamic non-bypassable list: Failed to parse nsagent message %q", line) continue } var newList []net.IPNet for _, intf := range msg.Interfaces { for _, cidr := range intf.CIDRs { _, ipNet, err := net.ParseCIDR(cidr) if err != nil { logrus.WithError(err).Warnf("Dynamic non-bypassable list: Failed to parse nsagent message %q: %q: bad CIDR %q", line, intf.Name, cidr) continue } if ipNet != nil { newList = append(newList, *ipNet) } } } x.mu.Lock() logrus.Infof("Dynamic non-bypassable list: old dynamic=%v, new dynamic=%v, static=%v", x.dynamicList, newList, x.staticList) x.dynamicList = newList x.mu.Unlock() } if err := scanner.Err(); err != nil { if !errors.Is(err, io.EOF) { logrus.WithError(err).Warn("Dynamic non-bypassable list: Error while parsing nsagent messages") } } } bypass4netns-0.4.1/pkg/bypass4netns/nsagent/000077500000000000000000000000001460472144300210335ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/nsagent/nsagent.go000066400000000000000000000027231460472144300230250ustar00rootroot00000000000000package nsagent import ( "encoding/json" "fmt" "io" "net" "os" "os/signal" "sort" "github.com/rootless-containers/bypass4netns/pkg/bypass4netns/nsagent/types" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) func Main() error { w := os.Stdout if err := inspect(w); err != nil { return err } sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, unix.SIGHUP, unix.SIGTERM, unix.SIGINT) for sig := range sigCh { switch sig { case unix.SIGHUP: if err := inspect(w); err != nil { return err } case unix.SIGTERM, unix.SIGINT: return nil } } return nil } func inspect(w io.Writer) error { var msg types.Message interfaces, err := net.Interfaces() if err != nil { return fmt.Errorf("failed to enumerate the network interfaces: %w", err) } for _, intf := range interfaces { addrs, err := intf.Addrs() if err != nil { logrus.Warnf("Failed to get the addresses of the network interface %q: %v", intf.Name, err) continue } entry := types.Interface{ Name: intf.Name, } for _, addr := range addrs { if ipNet, ok := addr.(*net.IPNet); ok { entry.CIDRs = append(entry.CIDRs, ipNet.String()) } } sort.Strings(entry.CIDRs) msg.Interfaces = append(msg.Interfaces, entry) } sort.Slice(msg.Interfaces, func(i, j int) bool { return msg.Interfaces[i].Name < msg.Interfaces[j].Name }) b, err := json.Marshal(msg) if err != nil { return err } b = append(b, []byte("\n")...) _, err = w.Write(b) return err } bypass4netns-0.4.1/pkg/bypass4netns/nsagent/types/000077500000000000000000000000001460472144300221775ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/nsagent/types/types.go000066400000000000000000000003511460472144300236710ustar00rootroot00000000000000package types type Message struct { Interfaces []Interface `json:"interfaces"` // sorted by Name } type Interface struct { Name string `json:"name"` // "lo", "eth0", etc. CIDRs []string `json:"cidrs"` // sorted as strings } bypass4netns-0.4.1/pkg/bypass4netns/seccomp.go000066400000000000000000000021771460472144300213630ustar00rootroot00000000000000package bypass4netns import ( "fmt" "syscall" "unsafe" libseccomp "github.com/seccomp/libseccomp-golang" "github.com/vtolstov/go-ioctl" ) const ( SeccompAddFdFlagSetFd = 1 SeccompUserNotifFlagContinue = 1 SeccompIocMagic = '!' ) func seccompIOW(nr, typ uintptr) uintptr { return ioctl.IOW(uintptr(SeccompIocMagic), nr, typ) } // C.SECCOMP_IOCTL_NOTIF_ADDFD become error // Error Message: could not determine kind of name for C.SECCOMP_IOCTL_NOTIF_ADDFD // TODO: use C.SECCOMP_IOCTL_NOTIF_ADDFD or add equivalent variable to libseccomp-go func seccompIoctlNotifAddfd() uintptr { return seccompIOW(3, uintptr(unsafe.Sizeof(seccompNotifAddFd{}))) } type seccompNotifAddFd struct { id uint64 flags uint32 srcfd uint32 newfd uint32 newfdFlags uint32 } func (addfd *seccompNotifAddFd) ioctlNotifAddFd(notifFd libseccomp.ScmpFd) error { ioctl_op := seccompIoctlNotifAddfd() _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(notifFd), ioctl_op, uintptr(unsafe.Pointer(addfd))) if errno != 0 { return fmt.Errorf("ioctl(SECCOMP_IOCTL_NOTIF_ADFD) failed: %s", errno) } return nil } bypass4netns-0.4.1/pkg/bypass4netns/sockaddr.go000066400000000000000000000051161460472144300215200ustar00rootroot00000000000000package bypass4netns import ( "bytes" "encoding/binary" "fmt" "net" "syscall" ) type sockaddr struct { syscall.RawSockaddr IP net.IP Port int Flowinfo uint32 // sin6_flowinfo ScopeID uint32 // sin6_scope_id } func (sa *sockaddr) String() string { return fmt.Sprintf("%s:%d", sa.IP, sa.Port) } func newSockaddr(buf []byte) (*sockaddr, error) { sa := &sockaddr{} reader := bytes.NewReader(buf) // TODO: support big endian hosts endian := binary.LittleEndian if err := binary.Read(reader, endian, &sa.RawSockaddr); err != nil { return nil, fmt.Errorf("cannot cast byte array to RawSocksddr: %w", err) } switch sa.Family { case syscall.AF_INET: addr4 := syscall.RawSockaddrInet4{} if _, err := reader.Seek(0, 0); err != nil { return nil, err } if err := binary.Read(reader, endian, &addr4); err != nil { return nil, fmt.Errorf("cannot cast byte array to RawSockaddrInet4: %w", err) } sa.IP = make(net.IP, len(addr4.Addr)) copy(sa.IP, addr4.Addr[:]) p := make([]byte, 2) binary.BigEndian.PutUint16(p, addr4.Port) sa.Port = int(endian.Uint16(p)) case syscall.AF_INET6: addr6 := syscall.RawSockaddrInet6{} if _, err := reader.Seek(0, 0); err != nil { return nil, err } if err := binary.Read(reader, endian, &addr6); err != nil { return nil, fmt.Errorf("cannot cast byte array to RawSockaddrInet6: %w", err) } sa.IP = make(net.IP, len(addr6.Addr)) copy(sa.IP, addr6.Addr[:]) p := make([]byte, 2) binary.BigEndian.PutUint16(p, addr6.Port) sa.Port = int(endian.Uint16(p)) sa.Flowinfo = addr6.Flowinfo sa.ScopeID = addr6.Scope_id default: return nil, fmt.Errorf("expected AF_INET or AF_INET6, got %d", sa.Family) } return sa, nil } func (sa *sockaddr) toBytes() ([]byte, error) { res := bytes.Buffer{} // TODO: support big endian hosts endian := binary.LittleEndian // ntohs p := make([]byte, 2) binary.BigEndian.PutUint16(p, uint16(sa.Port)) switch sa.Family { case syscall.AF_INET: addr4 := syscall.RawSockaddrInet4{} addr4.Family = syscall.AF_INET copy(addr4.Addr[:], sa.IP.To4()[:]) addr4.Port = endian.Uint16(p) err := binary.Write(&res, endian, addr4) if err != nil { return nil, err } case syscall.AF_INET6: addr6 := syscall.RawSockaddrInet6{} addr6.Family = syscall.AF_INET6 copy(addr6.Addr[:], sa.IP.To16()[:]) addr6.Port = endian.Uint16(p) addr6.Flowinfo = sa.Flowinfo addr6.Scope_id = sa.ScopeID err := binary.Write(&res, endian, addr6) if err != nil { return nil, err } default: return nil, fmt.Errorf("expected AF_INET or AF_INET6, got %d", sa.Family) } return res.Bytes(), nil } bypass4netns-0.4.1/pkg/bypass4netns/sockaddr_test.go000066400000000000000000000021161460472144300225540ustar00rootroot00000000000000package bypass4netns import ( "net" "syscall" "testing" "github.com/stretchr/testify/assert" ) func TestSerializeDeserializeSockaddr4(t *testing.T) { ip := net.ParseIP("192.168.1.100") port := 12345 sa := sockaddr{ IP: ip, Port: port, } sa.Family = syscall.AF_INET saBytes, err := sa.toBytes() assert.Equal(t, nil, err) assert.Equal(t, 16, len(saBytes)) sa2, err := newSockaddr(saBytes) assert.Equal(t, nil, err) assert.Equal(t, ip.String(), sa2.IP.String()) assert.Equal(t, port, sa2.Port) } func TestSerializeDeserializeSockaddr6(t *testing.T) { ip := net.ParseIP("2001:0db8::1:0:0:1") port := 12345 sa := sockaddr{ IP: ip, Port: port, Flowinfo: 0x12345678, ScopeID: 0x9abcdef0, } sa.Family = syscall.AF_INET6 saBytes, err := sa.toBytes() assert.Equal(t, nil, err) assert.Equal(t, 28, len(saBytes)) sa2, err := newSockaddr(saBytes) assert.Equal(t, nil, err) assert.Equal(t, ip.String(), sa2.IP.String()) assert.Equal(t, port, sa2.Port) assert.Equal(t, sa.Flowinfo, uint32(0x12345678)) assert.Equal(t, sa.ScopeID, uint32(0x9abcdef0)) } bypass4netns-0.4.1/pkg/bypass4netns/socket.go000066400000000000000000000304141460472144300212150ustar00rootroot00000000000000package bypass4netns import ( gocontext "context" "encoding/binary" "fmt" "net" "strconv" "strings" "syscall" "time" "unsafe" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) type socketOption struct { level uint64 optname uint64 optval []byte optlen uint64 } // Handle F_SETFL, F_SETFD options type fcntlOption struct { cmd uint64 value uint64 } type socketState int const ( // NotBypassableSocket means that the fd is not socket or not bypassed NotBypassable socketState = iota // NotBypassed means that the socket is not bypassed. NotBypassed // Bypassed means that the socket is replaced by one created on the host Bypassed // Error happened after bypass. Nothing can be done to recover from this state. Error ) func (ss socketState) String() string { switch ss { case NotBypassable: return "NotBypassable" case NotBypassed: return "NotBypassed" case Bypassed: return "Bypassed" case Error: return "Error" default: panic(fmt.Sprintf("unexpected enum %d: String() is not implmented", ss)) } } type processStatus struct { sockets map[int]*socketStatus } func newProcessStatus() *processStatus { return &processStatus{ sockets: map[int]*socketStatus{}, } } type socketStatus struct { state socketState pid int sockfd int sockDomain int sockType int sockProto int // address for bind or connect addr *sockaddr socketOptions []socketOption fcntlOptions []fcntlOption logger *logrus.Entry ignoreBind bool } func newSocketStatus(pid int, sockfd int, sockDomain, sockType, sockProto int, ignoreBind bool) *socketStatus { return &socketStatus{ state: NotBypassed, pid: pid, sockfd: sockfd, sockDomain: sockDomain, sockType: sockType, sockProto: sockProto, socketOptions: []socketOption{}, fcntlOptions: []fcntlOption{}, logger: logrus.WithFields(logrus.Fields{"pid": pid, "sockfd": sockfd}), ignoreBind: ignoreBind, } } func (ss *socketStatus) handleSysSetsockopt(pid int, handler *notifHandler, ctx *context) { ss.logger.Debug("handle setsockopt") level := ctx.req.Data.Args[1] optname := ctx.req.Data.Args[2] optlen := ctx.req.Data.Args[4] optval, err := handler.readProcMem(pid, ctx.req.Data.Args[3], optlen) if err != nil { ss.logger.Errorf("setsockopt readProcMem failed pid %v offset 0x%x: %s", pid, ctx.req.Data.Args[1], err) } value := socketOption{ level: level, optname: optname, optval: optval, optlen: optlen, } ss.socketOptions = append(ss.socketOptions, value) ss.logger.Debugf("setsockopt level=%d optname=%d optval=%v optlen=%d was recorded.", level, optname, optval, optlen) } func (ss *socketStatus) handleSysFcntl(ctx *context) { ss.logger.Debug("handle fcntl") fcntlCmd := ctx.req.Data.Args[1] switch fcntlCmd { case unix.F_SETFD: // 0x2 case unix.F_SETFL: // 0x4 opt := fcntlOption{ cmd: fcntlCmd, value: ctx.req.Data.Args[2], } ss.fcntlOptions = append(ss.fcntlOptions, opt) ss.logger.Debugf("fcntl cmd=0x%x value=%d was recorded.", fcntlCmd, opt.value) case unix.F_GETFL: // 0x3 // ignore these default: ss.logger.Warnf("Unknown fcntl command 0x%x ignored.", fcntlCmd) } } func (ss *socketStatus) handleSysConnect(handler *notifHandler, ctx *context) { destAddr, err := handler.readSockaddrFromProcess(ss.pid, ctx.req.Data.Args[1], ctx.req.Data.Args[2]) if err != nil { ss.logger.Errorf("failed to read sockaddr from process: %q", err) return } ss.addr = destAddr var newDestAddr net.IP switch destAddr.Family { case syscall.AF_INET: newDestAddr = net.IPv4zero newDestAddr = newDestAddr.To4() newDestAddr[0] = 127 newDestAddr[3] = 1 case syscall.AF_INET6: newDestAddr = net.IPv6loopback newDestAddr = newDestAddr.To16() default: ss.logger.Errorf("unexpected destination address family %d", destAddr.Family) ss.state = Error return } // check whether the destination is bypassed or not. connectToLoopback := false connectToInterface := false connectToOtherBypassedContainer := false var fwdPort ForwardPortMapping if !ss.ignoreBind { var ok bool fwdPort, ok = handler.forwardingPorts[int(destAddr.Port)] if ok { if destAddr.IP.IsLoopback() { ss.logger.Infof("destination address %v is loopback and bypassed", destAddr) connectToLoopback = true } else if contIf, ok := handler.containerInterfaces[destAddr.String()]; ok && contIf.containerID == handler.state.State.ID { ss.logger.Infof("destination address %v is interface's address and bypassed", destAddr) connectToInterface = true } } } if handler.multinode.Enable && destAddr.IP.IsPrivate() { // currently, only private addresses are available in multinode communication. key := ETCD_MULTINODE_PREFIX + destAddr.String() ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 2*time.Second) res, err := handler.multinode.etcdClient.Get(ctx, ETCD_MULTINODE_PREFIX+destAddr.String()) cancel() if err != nil { ss.logger.WithError(err).Warnf("destination address %q is not registered", key) } else { if len(res.Kvs) != 1 { ss.logger.Errorf("invalid len(res.Kvs) %d", len(res.Kvs)) ss.state = Error return } hostAddrWithPort := string(res.Kvs[0].Value) hostAddrs := strings.Split(hostAddrWithPort, ":") if len(hostAddrs) != 2 { ss.logger.Errorf("invalid address format %q", hostAddrWithPort) ss.state = Error return } hostAddr := hostAddrs[0] hostPort, err := strconv.Atoi(hostAddrs[1]) if err != nil { ss.logger.Errorf("invalid address format %q", hostAddrWithPort) ss.state = Error return } newDestAddr = net.ParseIP(hostAddr) fwdPort.HostPort = hostPort connectToOtherBypassedContainer = true ss.logger.Infof("destination address %v is container address and bypassed via overlay network", destAddr) } } else if handler.c2cConnections.Enable { contIf, ok := handler.containerInterfaces[destAddr.String()] if ok { ss.logger.Infof("destination address %v is container address and bypassed", destAddr) fwdPort.HostPort = contIf.hostPort connectToOtherBypassedContainer = true } } // check whether the destination container socket is bypassed or not. isNotBypassed := handler.nonBypassable.Contains(destAddr.IP) if !connectToLoopback && !connectToInterface && !connectToOtherBypassedContainer && isNotBypassed { ss.logger.Infof("destination address %v is not bypassed.", destAddr.IP) ss.state = NotBypassable return } sockfdOnHost, err := syscall.Socket(ss.sockDomain, ss.sockType, ss.sockProto) if err != nil { ss.logger.Errorf("failed to create socket: %q", err) ss.state = NotBypassable return } defer syscall.Close(sockfdOnHost) err = ss.configureSocket(sockfdOnHost) if err != nil { ss.logger.Errorf("failed to configure socket: %q", err) ss.state = NotBypassable return } addfd := seccompNotifAddFd{ id: ctx.req.ID, flags: SeccompAddFdFlagSetFd, srcfd: uint32(sockfdOnHost), newfd: uint32(ctx.req.Data.Args[0]), newfdFlags: 0, } err = addfd.ioctlNotifAddFd(ctx.notifFd) if err != nil { ss.logger.Errorf("ioctl NotifAddFd failed: %q", err) ss.state = NotBypassable return } if connectToLoopback || connectToInterface || connectToOtherBypassedContainer { p := make([]byte, 2) binary.BigEndian.PutUint16(p, uint16(fwdPort.HostPort)) // writing host port at sock_addr's port offset // TODO: should we return dummy value when getpeername(2) is called? err = handler.writeProcMem(ss.pid, ctx.req.Data.Args[1]+2, p) if err != nil { ss.logger.Errorf("failed to rewrite destination port: %q", err) ss.state = Error return } ss.logger.Infof("destination's port %d is rewritten to host-side port %d", ss.addr.Port, fwdPort.HostPort) } if connectToInterface || connectToOtherBypassedContainer { // writing host's loopback address to connect to bypassed socket at sock_addr's address offset // TODO: should we return dummy value when getpeername(2) is called? switch destAddr.Family { case syscall.AF_INET: newDestAddr = newDestAddr.To4() err = handler.writeProcMem(ss.pid, ctx.req.Data.Args[1]+4, newDestAddr[0:4]) case syscall.AF_INET6: newDestAddr = newDestAddr.To16() err = handler.writeProcMem(ss.pid, ctx.req.Data.Args[1]+8, newDestAddr[0:16]) default: ss.logger.Errorf("unexpected destination address family %d", destAddr.Family) ss.state = Error return } if err != nil { ss.logger.Errorf("failed to rewrite destination address: %q", err) ss.state = Error return } ss.logger.Infof("destination address %s is rewritten to %s", destAddr.IP, newDestAddr) } ss.state = Bypassed ss.logger.Infof("bypassed connect socket destAddr=%s", ss.addr) } func (ss *socketStatus) handleSysBind(pid int, handler *notifHandler, ctx *context) { if ss.ignoreBind { ss.state = NotBypassable return } sa, err := handler.readSockaddrFromProcess(pid, ctx.req.Data.Args[1], ctx.req.Data.Args[2]) if err != nil { ss.logger.Errorf("failed to read sockaddr from process: %q", err) ss.state = NotBypassable return } ss.addr = sa ss.logger.Infof("handle port=%d, ip=%v", sa.Port, sa.IP) // TODO: get port-fowrad mapping from nerdctl fwdPort, ok := handler.forwardingPorts[int(sa.Port)] if !ok { ss.logger.Infof("port=%d is not target of port forwarding.", sa.Port) ss.state = NotBypassable return } sockfdOnHost, err := syscall.Socket(ss.sockDomain, ss.sockType, ss.sockProto) if err != nil { ss.logger.Errorf("failed to create socket: %q", err) ss.state = NotBypassable return } defer syscall.Close(sockfdOnHost) err = ss.configureSocket(sockfdOnHost) if err != nil { ss.logger.Errorf("failed to configure socket: %q", err) ss.state = NotBypassable return } var bind_addr syscall.Sockaddr switch sa.Family { case syscall.AF_INET: var addr [4]byte for i := 0; i < 4; i++ { addr[i] = sa.IP[i] } bind_addr = &syscall.SockaddrInet4{ Port: fwdPort.HostPort, Addr: addr, } case syscall.AF_INET6: var addr [16]byte for i := 0; i < 16; i++ { addr[i] = sa.IP[i] } bind_addr = &syscall.SockaddrInet6{ Port: fwdPort.HostPort, ZoneId: sa.ScopeID, Addr: addr, } } err = syscall.Bind(sockfdOnHost, bind_addr) if err != nil { ss.logger.Errorf("bind failed: %s", err) ss.state = NotBypassable return } addfd := seccompNotifAddFd{ id: ctx.req.ID, flags: SeccompAddFdFlagSetFd, srcfd: uint32(sockfdOnHost), newfd: uint32(ctx.req.Data.Args[0]), newfdFlags: 0, } err = addfd.ioctlNotifAddFd(ctx.notifFd) if err != nil { ss.logger.Errorf("ioctl NotifAddFd failed: %s", err) ss.state = NotBypassable return } ss.state = Bypassed ss.logger.Infof("bypassed bind socket for %d:%d is done", fwdPort.HostPort, fwdPort.ChildPort) ctx.resp.Flags &= (^uint32(SeccompUserNotifFlagContinue)) } func (ss *socketStatus) handleSysGetpeername(handler *notifHandler, ctx *context) { if ss.addr == nil { return } buf, err := ss.addr.toBytes() if err != nil { ss.logger.WithError(err).Errorf("failed to serialize address %s", ss.addr) return } err = handler.writeProcMem(ss.pid, ctx.req.Data.Args[1], buf) if err != nil { ss.logger.WithError(err).Errorf("failed to write address %s", ss.addr) return } bufLen := make([]byte, 4) binary.LittleEndian.PutUint32(bufLen, uint32(len(buf))) err = handler.writeProcMem(ss.pid, ctx.req.Data.Args[2], bufLen) if err != nil { ss.logger.WithError(err).Errorf("failed to write address length %d", len(buf)) return } ctx.resp.Flags &= (^uint32(SeccompUserNotifFlagContinue)) ss.logger.Infof("rewrite getpeername() address to %s", ss.addr) } func (ss *socketStatus) configureSocket(sockfd int) error { for _, optVal := range ss.socketOptions { _, _, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, uintptr(sockfd), uintptr(optVal.level), uintptr(optVal.optname), uintptr(unsafe.Pointer(&optVal.optval[0])), uintptr(optVal.optlen), 0) if errno != 0 { return fmt.Errorf("setsockopt failed(%v): %s", optVal, errno) } ss.logger.Debugf("configured socket option val=%v", optVal) } for _, fcntlVal := range ss.fcntlOptions { _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(sockfd), uintptr(fcntlVal.cmd), uintptr(fcntlVal.value)) if errno != 0 { return fmt.Errorf("fnctl failed(%v): %s", fcntlVal, errno) } ss.logger.Debugf("configured socket fcntl val=%v", fcntlVal) } return nil } bypass4netns-0.4.1/pkg/bypass4netns/tracer/000077500000000000000000000000001460472144300206545ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netns/tracer/agent.go000066400000000000000000000042531460472144300223050ustar00rootroot00000000000000package tracer import ( "encoding/json" "fmt" "net" "os" "time" "github.com/sirupsen/logrus" ) type TracerCommand struct { Cmd TracerCmd `json:"tracerCmd"` ForwardingPorts []int `json:"forwardingPorts,omitempty"` DestinationAddress []string `json:"destinationAddress,omitempty"` } type TracerCmd int const ( Ok TracerCmd = iota RegisterForwardPorts ConnectToAddress ) func Main() error { r := os.Stdin w := os.Stdout dec := json.NewDecoder(r) for { var cmd TracerCommand err := dec.Decode(&cmd) if err != nil { logrus.WithError(err).Errorf("failed to decode") break } logrus.Infof("decoded = %v", cmd) switch cmd.Cmd { case RegisterForwardPorts: for _, p := range cmd.ForwardingPorts { readyChan := make(chan bool) go func(port int, c chan bool) { err := listenLoop(port, c) if err != nil { logrus.WithError(err).Errorf("failed to listen on port %d", port) } }(p, readyChan) <-readyChan } cmd = TracerCommand{ Cmd: Ok, } case ConnectToAddress: addrs := []string{} for i := range cmd.DestinationAddress { addr := cmd.DestinationAddress[i] err = tryToConnect(addr) if err != nil { logrus.WithError(err).Warnf("failed to connect to %s", addr) continue } addrs = append(addrs, addr) } cmd = TracerCommand{ Cmd: Ok, DestinationAddress: addrs, } } m, err := json.Marshal(cmd) if err != nil { logrus.WithError(err).Errorf("failed to encode") } _, err = w.Write(m) if err != nil { logrus.WithError(err).Errorf("failed to write") } } logrus.Infof("Exit.") return nil } func listenLoop(port int, readyChan chan bool) error { l, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return err } defer l.Close() readyChan <- true logrus.Infof("started to listen on port %d", port) for { conn, err := l.Accept() if err != nil { return err } conn.Close() } } func tryToConnect(addr string) error { conn, err := net.DialTimeout("tcp", addr, 10*time.Millisecond) if err != nil { return err } defer conn.Close() logrus.Infof("successfully connected to %s", addr) return nil } bypass4netns-0.4.1/pkg/bypass4netns/tracer/tracer.go000066400000000000000000000055161460472144300224720ustar00rootroot00000000000000package tracer import ( "context" "encoding/json" "fmt" "io" "os" "os/exec" "strconv" "sync" "github.com/rootless-containers/bypass4netns/pkg/util" "golang.org/x/sys/unix" ) type Tracer struct { logPath string tracerCmd *exec.Cmd reader io.Reader writer io.Writer lock sync.Mutex } func NewTracer(logPath string) *Tracer { return &Tracer{ logPath: logPath, lock: sync.Mutex{}, } } // StartTracer starts tracer in NS associated with the PID. func (x *Tracer) StartTracer(ctx context.Context, pid int) error { selfExe, err := os.Executable() if err != nil { return err } nsenter, err := exec.LookPath("nsenter") if err != nil { return err } nsenterFlags := []string{ "-t", strconv.Itoa(pid), "-F", "-n", } selfPid := os.Getpid() ok, err := util.SameUserNS(pid, selfPid) if err != nil { return fmt.Errorf("failed to check sameUserNS(%d, %d)", pid, selfPid) } if !ok { nsenterFlags = append(nsenterFlags, "-U", "--preserve-credentials") } nsenterFlags = append(nsenterFlags, "--", selfExe, "--tracer-agent", "--log-file", x.logPath) x.tracerCmd = exec.CommandContext(ctx, nsenter, nsenterFlags...) x.tracerCmd.SysProcAttr = &unix.SysProcAttr{ Pdeathsig: unix.SIGTERM, } x.tracerCmd.Stderr = os.Stderr x.reader, x.tracerCmd.Stdout = io.Pipe() x.tracerCmd.Stdin, x.writer = io.Pipe() if err := x.tracerCmd.Start(); err != nil { return fmt.Errorf("failed to start %v: %w", x.tracerCmd.Args, err) } return nil } func (x *Tracer) RegisterForwardPorts(ports []int) error { cmd := TracerCommand{ Cmd: RegisterForwardPorts, ForwardingPorts: ports, } m, err := json.Marshal(cmd) if err != nil { return err } writeSize, err := x.writer.Write(m) if err != nil { return err } if writeSize != len(m) { return fmt.Errorf("unexpected written size expected=%d actual=%d", len(m), writeSize) } dec := json.NewDecoder(x.reader) var resp TracerCommand err = dec.Decode(&resp) if err != nil { return fmt.Errorf("invalid response: %q", err) } if resp.Cmd != Ok { return fmt.Errorf("unexpected response: %d", resp.Cmd) } return nil } func (x *Tracer) ConnectToAddress(addrs []string) ([]string, error) { x.lock.Lock() defer x.lock.Unlock() cmd := TracerCommand{ Cmd: ConnectToAddress, DestinationAddress: addrs, } m, err := json.Marshal(cmd) if err != nil { return nil, err } writeSize, err := x.writer.Write(m) if err != nil { return nil, err } if writeSize != len(m) { return nil, fmt.Errorf("unexpected written size expected=%d actual=%d", len(m), writeSize) } dec := json.NewDecoder(x.reader) var resp TracerCommand err = dec.Decode(&resp) if err != nil { return nil, fmt.Errorf("invalid response: %q", err) } if resp.Cmd != Ok { return nil, fmt.Errorf("unexpected response: %d", resp.Cmd) } return resp.DestinationAddress, nil } bypass4netns-0.4.1/pkg/bypass4netnsd/000077500000000000000000000000001460472144300175405ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/bypass4netnsd/bypass4netnsd.go000066400000000000000000000147641460472144300227040ustar00rootroot00000000000000package bypass4netnsd import ( "errors" "fmt" "os" "os/exec" "sync" "syscall" "time" "github.com/rootless-containers/bypass4netns/pkg/api" "github.com/rootless-containers/bypass4netns/pkg/api/com" "github.com/rootless-containers/bypass4netns/pkg/util" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) type Driver struct { BypassExecutablePath string ComSocketPath string bypass map[string]api.BypassStatus lock sync.RWMutex containerInterfaces map[string]com.ContainerInterfaces interfacesLock sync.RWMutex HandleC2CEnable bool TracerEnable bool MultinodeEnable bool MultinodeEtcdAddress string MultinodeHostAddress string } func NewDriver(execPath string, comSocketPath string) *Driver { return &Driver{ BypassExecutablePath: execPath, ComSocketPath: comSocketPath, bypass: map[string]api.BypassStatus{}, lock: sync.RWMutex{}, containerInterfaces: map[string]com.ContainerInterfaces{}, interfacesLock: sync.RWMutex{}, TracerEnable: false, MultinodeEnable: false, } } func (d *Driver) ListBypass() []api.BypassStatus { d.lock.RLock() defer d.lock.RUnlock() res := []api.BypassStatus{} for _, v := range d.bypass { res = append(res, v) } return res } func (d *Driver) StartBypass(spec *api.BypassSpec) (*api.BypassStatus, error) { logger := logrus.WithFields(logrus.Fields{"ID": util.ShrinkID(spec.ID)}) logger.Info("Starting bypass") b4nnArgs := []string{} if logger.Logger.GetLevel() == logrus.DebugLevel { b4nnArgs = append(b4nnArgs, "--debug") } if spec.SocketPath != "" { socketOption := fmt.Sprintf("--socket=%s", spec.SocketPath) b4nnArgs = append(b4nnArgs, socketOption) } if spec.PidFilePath != "" { pidFileOption := fmt.Sprintf("--pid-file=%s", spec.PidFilePath) b4nnArgs = append(b4nnArgs, pidFileOption) } if spec.LogFilePath != "" { logFileOption := fmt.Sprintf("--log-file=%s", spec.LogFilePath) b4nnArgs = append(b4nnArgs, logFileOption) } for _, port := range spec.PortMapping { b4nnArgs = append(b4nnArgs, fmt.Sprintf("-p=%d:%d", port.ParentPort, port.ChildPort)) } for _, subnet := range spec.IgnoreSubnets { b4nnArgs = append(b4nnArgs, fmt.Sprintf("--ignore=%s", subnet)) } if spec.IgnoreBind { b4nnArgs = append(b4nnArgs, "--ignore-bind") } b4nnArgs = append(b4nnArgs, fmt.Sprintf("--com-socket=%s", d.ComSocketPath)) if d.HandleC2CEnable { b4nnArgs = append(b4nnArgs, "--handle-c2c-connections") } if d.TracerEnable { b4nnArgs = append(b4nnArgs, "--tracer=true") } if d.MultinodeEnable { b4nnArgs = append(b4nnArgs, "--multinode=true") b4nnArgs = append(b4nnArgs, fmt.Sprintf("--multinode-etcd-address=%s", d.MultinodeEtcdAddress)) b4nnArgs = append(b4nnArgs, fmt.Sprintf("--multinode-host-address=%s", d.MultinodeHostAddress)) } // prepare pipe for ready notification readyR, readyW, err := os.Pipe() if err != nil { return nil, err } defer readyR.Close() defer readyW.Close() // fd in b4nnCmd.ExtraFiles is assigned in child process from fd=3 readyFdOption := "--ready-fd=3" b4nnArgs = append(b4nnArgs, readyFdOption) logger.Infof("bypass4netns args:%v", b4nnArgs) b4nnCmd := exec.Command(d.BypassExecutablePath, b4nnArgs...) b4nnCmd.ExtraFiles = append(b4nnCmd.ExtraFiles, readyW) err = b4nnCmd.Start() if err != nil { return nil, err } err = waitForReadyFD(b4nnCmd.Process.Pid, readyR) if err != nil { return nil, err } logger.Info("bypass4netns successfully started") d.lock.Lock() defer d.lock.Unlock() status := api.BypassStatus{ ID: spec.ID, Pid: b4nnCmd.Process.Pid, Spec: *spec, } d.bypass[status.ID] = status logger.Info("Started bypass") return &status, nil } func (d *Driver) StopBypass(id string) error { logger := logrus.WithFields(logrus.Fields{"ID": util.ShrinkID(id)}) logger.Infof("Stopping bypass") d.lock.Lock() defer d.lock.Unlock() bStatus, ok := d.bypass[id] if !ok { return fmt.Errorf("child %s not found", id) } proc, err := os.FindProcess(bStatus.Pid) if err != nil { return err } logger.Debugf("bypass4netns found pid=%d", proc.Pid) logger.Infof("Terminating bypass4netns pid=%d", proc.Pid) err = proc.Signal(unix.SIGTERM) if err != nil { return err } // wait for the process exit // TODO: Timeout if _, err := proc.Wait(); err != nil { logrus.Warnf("Failed to terminate bypass4netns pid=%d with SIGTERM, killing...", proc.Pid) err = proc.Kill() if err != nil { return err } _, _ = proc.Wait() } logger.Infof("Terminated bypass4netns pid=%d", proc.Pid) delete(d.bypass, id) logger.Info("Stopped bypass") // remove the container's interfaces d.DeleteInterface(id) return nil } func (d *Driver) ListInterfaces() map[string]com.ContainerInterfaces { d.interfacesLock.RLock() defer d.interfacesLock.RUnlock() ifs := map[string]com.ContainerInterfaces{} // copy map for k := range d.containerInterfaces { ifs[k] = d.containerInterfaces[k] } return ifs } func (d *Driver) GetInterface(id string) *com.ContainerInterfaces { d.interfacesLock.RLock() defer d.interfacesLock.RUnlock() ifs, ok := d.containerInterfaces[id] if !ok { return nil } return &ifs } func (d *Driver) PostInterface(id string, containerIfs *com.ContainerInterfaces) { d.interfacesLock.Lock() defer d.interfacesLock.Unlock() d.containerInterfaces[id] = *containerIfs } func (d *Driver) DeleteInterface(id string) { d.interfacesLock.Lock() defer d.interfacesLock.Unlock() delete(d.containerInterfaces, id) } // waitForReady is from libpod // https://github.com/containers/libpod/blob/e6b843312b93ddaf99d0ef94a7e60ff66bc0eac8/libpod/networking_linux.go#L272-L308 func waitForReadyFD(cmdPid int, r *os.File) error { b := make([]byte, 16) for { if err := r.SetDeadline(time.Now().Add(1 * time.Second)); err != nil { return fmt.Errorf("error setting bypass4netns pipe timeout: %w", err) } if _, err := r.Read(b); err == nil { break } else { if os.IsTimeout(err) { // Check if the process is still running. var status syscall.WaitStatus pid, err := syscall.Wait4(cmdPid, &status, syscall.WNOHANG, nil) if err != nil { return fmt.Errorf("failed to read bypass4netns process status: %w", err) } if pid != cmdPid { continue } if status.Exited() { return errors.New("bypass4netns failed") } if status.Signaled() { return errors.New("bypass4netns killed by signal") } continue } return fmt.Errorf("failed to read from bypass4netns sync pipe: %w", err) } } return nil } bypass4netns-0.4.1/pkg/oci/000077500000000000000000000000001460472144300155115ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/oci/oci.go000066400000000000000000000030751460472144300166170ustar00rootroot00000000000000package oci import ( "fmt" "reflect" "github.com/opencontainers/runtime-spec/specs-go" ) const ( SocketName = "bypass4netns.sock" ) var SyscallsToBeNotified = []string{"bind", "close", "connect", "setsockopt", "fcntl", "_exit", "exit_group", "getpeername"} func GetDefaultSeccompProfile(listenerPath string) *specs.LinuxSeccomp { tmpl := specs.LinuxSeccomp{ DefaultAction: specs.ActAllow, } seccomp, err := TranslateSeccompProfile(tmpl, listenerPath) if err != nil { panic(err) } return seccomp } func TranslateSeccompProfile(old specs.LinuxSeccomp, listenerPath string) (*specs.LinuxSeccomp, error) { sc := old if sc.ListenerPath != "" && sc.ListenerPath != listenerPath { return nil, fmt.Errorf("bypass4netns's seccomp listener path %q conflicts with the existing seccomp listener path %q", listenerPath, sc.ListenerPath) } sc.ListenerPath = listenerPath prepend := specs.LinuxSyscall{ Names: SyscallsToBeNotified, Action: specs.ActNotify, } if alreadyPrepended := len(sc.Syscalls) > 0 && reflect.DeepEqual(sc.Syscalls[0], prepend); !alreadyPrepended { for i := range sc.Syscalls { i := i sc.Syscalls[i].Names = filterStringSlice(sc.Syscalls[i].Names, SyscallsToBeNotified) } sc.Syscalls = append([]specs.LinuxSyscall{prepend}, sc.Syscalls...) } return &sc, nil } func filterStringSlice(ss, banned []string) []string { bannedM := make(map[string]struct{}, len(banned)) for _, f := range banned { bannedM[f] = struct{}{} } var res []string for _, s := range ss { if _, ok := bannedM[s]; !ok { res = append(res, s) } } return res } bypass4netns-0.4.1/pkg/util/000077500000000000000000000000001460472144300157145ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/util/util.go000066400000000000000000000035231460472144300172230ustar00rootroot00000000000000package util import ( "fmt" "net" "os" "syscall" ) // shrinkID shrinks id to short(12 chars) id // 6d9bcda7cebd551ddc9e3173d2139386e21b56b241f8459c950ef58e036f6bd8 // to // 6d9bcda7cebd func ShrinkID(id string) string { if len(id) < 12 { return id } return id[0:12] } func SameUserNS(pidX, pidY int) (bool, error) { nsX := fmt.Sprintf("/proc/%d/ns/user", pidX) nsY := fmt.Sprintf("/proc/%d/ns/user", pidY) nsXResolved, err := os.Readlink(nsX) if err != nil { return false, err } nsYResolved, err := os.Readlink(nsY) if err != nil { return false, err } return nsXResolved == nsYResolved, nil } // copied from https://github.com/pfnet-research/meta-fuse-csi-plugin/blob/437dbbbbf16e5b02f9a508e3403d044b0a9dff89/pkg/util/fdchannel.go#L29 // which is licensed under apache 2.0 func SendMsg(via net.Conn, fd int, msg []byte) error { conn, ok := via.(*net.UnixConn) if !ok { return fmt.Errorf("failed to cast via to *net.UnixConn") } connf, err := conn.File() if err != nil { return err } socket := int(connf.Fd()) defer connf.Close() rights := syscall.UnixRights(fd) return syscall.Sendmsg(socket, msg, rights, nil, 0) } func RecvMsg(via net.Conn) (int, []byte, error) { conn, ok := via.(*net.UnixConn) if !ok { return 0, nil, fmt.Errorf("failed to cast via to *net.UnixConn") } connf, err := conn.File() if err != nil { return 0, nil, err } socket := int(connf.Fd()) defer connf.Close() buf := make([]byte, syscall.CmsgSpace(4)) b := make([]byte, 500) //nolint:dogsled n, _, _, _, err := syscall.Recvmsg(socket, b, buf, 0) if err != nil { return 0, nil, err } var msgs []syscall.SocketControlMessage msgs, err = syscall.ParseSocketControlMessage(buf) if err != nil { return 0, nil, err } fds, err := syscall.ParseUnixRights(&msgs[0]) if err != nil { return 0, nil, err } return fds[0], b[:n], err } bypass4netns-0.4.1/pkg/version/000077500000000000000000000000001460472144300164245ustar00rootroot00000000000000bypass4netns-0.4.1/pkg/version/version.go000066400000000000000000000001501460472144300204340ustar00rootroot00000000000000package version var ( // Version is filled on compilation time Version = "" ) bypass4netns-0.4.1/test/000077500000000000000000000000001460472144300151355ustar00rootroot00000000000000bypass4netns-0.4.1/test/Dockerfile000066400000000000000000000003531460472144300171300ustar00rootroot00000000000000FROM public.ecr.aws/docker/library/alpine:3.16 RUN apk add python3 ADD ./test_connect.py /tmp/test_connect.py ADD ./test_sendto.py /tmp/test_sendto.py ADD ./test_sendmsg.py /tmp/test_sendmsg.py CMD ["/bin/sh" "-c" "sleep infinity"] bypass4netns-0.4.1/test/DockerfileHttpServer000066400000000000000000000004331460472144300211560ustar00rootroot00000000000000FROM golang:1.22.0 as bench-builder COPY httpserver.go . # static link RUN CGO_ENABLED=0 go build -o /httpserver httpserver.go FROM ubuntu:22.04 RUN apt-get update && apt-get upgrade -y COPY --from=bench-builder /httpserver /httpserver CMD ["/bin/bash", "-c", "sleep infinity"] bypass4netns-0.4.1/test/enter.sh000077500000000000000000000002031460472144300166040ustar00rootroot00000000000000#!/bin/bash set -eux -o pipefail PID=$(nerdctl inspect $1 | jq '.[0].State.Pid') nsenter -t $PID -F -U --preserve-credentials -n bypass4netns-0.4.1/test/export_lxc_image.sh000077500000000000000000000003451460472144300210270ustar00rootroot00000000000000#!/bin/bash set -eux -o pipefail IMAGE_NAME=$1 sudo lxc snapshot $IMAGE_NAME snp0 sudo lxc publish $IMAGE_NAME/snp0 --alias $IMAGE_NAME-export --compression zstd sudo lxc image export $IMAGE_NAME-export /tmp/$IMAGE_NAME-image bypass4netns-0.4.1/test/httpserver.go000066400000000000000000000022211460472144300176670ustar00rootroot00000000000000package main import ( "bytes" "flag" "fmt" "io" "log" "net/http" ) var ( url = flag.String("url", "http://localhost/blk-1m", "") mode = flag.String("mode", "server", "") ) func main() { flag.Parse() // disable connection pool http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = -1 if *mode == "server" { fmt.Println("starting server") server() } else if *mode == "client" { err := client(*url) if err != nil { log.Fatal(err) } } } func server() { http.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "pong") }) log.Fatal(http.ListenAndServe(":8080", nil)) } func client(url string) error { resp, err := http.Get(url) if err != nil { return fmt.Errorf("failed Do err=%q", err) } if resp.StatusCode != 200 { return fmt.Errorf("unexpected status code %d", resp.StatusCode) } else { var buffer bytes.Buffer _, err = io.Copy(&buffer, resp.Body) if err != nil { return fmt.Errorf("failed Copy() err=%q", err) } fmt.Printf("resp=%s\n", buffer.String()) } err = resp.Body.Close() if err != nil { return fmt.Errorf("failed Close() err=%q", err) } return nil } bypass4netns-0.4.1/test/init_test.sh000077500000000000000000000036351460472144300175050ustar00rootroot00000000000000#!/bin/bash TEST_USER=ubuntu if [ -v GITHUB_WORKSPACE ]; then TEST_USER=runner fi set -eu -o pipefail if [ "$(whoami)" != "$TEST_USER" ]; then su $TEST_USER -c $0 exit 0 fi GO_VERSION="1.22.0" NERDCTL_VERSION="2.0.0-beta.3" echo "===== Prepare =====" ( set -x # for lxc if [ -d /host ]; then sudo cp -r /host ~/bypass4netns fi # for github actions runner if [ $TEST_USER == "runner" ]; then cd ../ cp -r bypass4netns ~/bypass4netns fi sudo chown -R $TEST_USER:$TEST_USER ~/bypass4netns sudo apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y build-essential curl dbus-user-session iperf3 libseccomp-dev uidmap python3 pkg-config iptables etcd jq tcpdump ethtool python3-pip git pip3 install matplotlib numpy sudo systemctl stop etcd sudo systemctl disable etcd systemctl --user start dbus curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz | sudo tar Cxz /usr/local echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.profile source ~/.profile curl -fsSL https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-full-${NERDCTL_VERSION}-linux-amd64.tar.gz | sudo tar Cxz /usr/local containerd-rootless-setuptool.sh install containerd-rootless-setuptool.sh install-buildkit # build nerdctl with bypass4netns curl -fsSL https://github.com/containerd/nerdctl/archive/refs/tags/v${NERDCTL_VERSION}.tar.gz | tar Cxz ~/ cd ~/nerdctl-${NERDCTL_VERSION} echo "replace github.com/rootless-containers/bypass4netns => /home/$TEST_USER/bypass4netns" >> go.mod go mod tidy make sudo rm -f /usr/local/bin/nerdctl sudo cp _output/nerdctl /usr/local/bin/nerdctl nerdctl info cd ~/bypass4netns make sudo rm -f /usr/local/bin/bypass4netns* sudo make install # also enable rootful containerd for rootful container testing sudo systemctl enable --now containerd sudo systemctl enable --now buildkit ) bypass4netns-0.4.1/test/launch_test_lxc.sh000077500000000000000000000011351460472144300206530ustar00rootroot00000000000000#!/bin/bash set -ux -o pipefail IMAGE=${1:-"images:ubuntu/22.04"} cd $(dirname $0) # lxd init --auto --storage-backend=btrfs sudo lxc launch -c security.privileged=true -c security.nesting=true $IMAGE test sudo lxc config device add test share disk source=$(cd ../; pwd) path=/host sudo lxc exec test -- /bin/bash -c "echo 'ubuntu ALL=NOPASSWD: ALL' | EDITOR='tee -a' visudo" # let user services running # this sometimes fails, retry until success RES=1 while [ $RES -ne 0 ] do sleep 1 sudo lxc exec test -- sudo --login --user ubuntu /bin/bash -c "sudo loginctl enable-linger" RES=$? done bypass4netns-0.4.1/test/lxd.yaml000066400000000000000000000013271460472144300166130ustar00rootroot00000000000000config: {} networks: - config: ipv4.address: 192.168.6.1/24 ipv4.nat: "true" ipv6.address: none description: "" name: lxdbr0 type: bridge project: default storage_pools: - config: size: 30GiB source: /var/snap/lxd/common/lxd/disks/default.img description: "" name: default driver: btrfs profiles: - config: {} description: Default LXD profile devices: eth0: name: eth0 network: lxdbr0 type: nic root: path: / pool: default type: disk name: default projects: - config: features.images: "true" features.networks: "true" features.profiles: "true" features.storage.volumes: "true" description: Default LXD project name: defaultbypass4netns-0.4.1/test/multinode.sh000077500000000000000000000060121460472144300174730ustar00rootroot00000000000000#!/bin/bash cd $(dirname $0) . ./util.sh set +e NAME="test" exec_lxc nerdctl rm -f vxlan sudo lxc rm -f test2 TEST1_VXLAN_MAC="02:42:c0:a8:00:1" TEST1_VXLAN_ADDR="192.168.2.1" TEST2_VXLAN_MAC="02:42:c0:a8:00:2" TEST2_VXLAN_ADDR="192.168.2.2" ALPINE_IMAGE="public.ecr.aws/docker/library/alpine:3.16" set -eux -o pipefail sudo lxc stop test sudo lxc copy test test2 sudo lxc start test sudo lxc start test2 sleep 5 TEST_ADDR=$(sudo lxc exec test -- hostname -I | sed 's/ //') TEST2_ADDR=$(sudo lxc exec test2 -- hostname -I | sed 's/ //') echo "===== Benchmark: iperf3 client(w/o bypass4netns) server(w/o bypass4netns) with multinode via VXLAN =====" ( NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name vxlan -d $ALPINE_IMAGE sleep infinity" NAME="test" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh vxlan $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR $TEST2_ADDR $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR NAME="test" exec_lxc nerdctl exec vxlan apk add --no-cache iperf3 NAME="test" exec_lxc systemd-run --user --unit run-test-iperf3 nerdctl exec vxlan iperf3 -s NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run -p 4789:4789/udp --privileged --name vxlan -d $ALPINE_IMAGE sleep infinity" NAME="test2" exec_lxc /home/ubuntu/bypass4netns/test/setup_vxlan.sh vxlan $TEST2_VXLAN_MAC $TEST2_VXLAN_ADDR $TEST_ADDR $TEST1_VXLAN_MAC $TEST1_VXLAN_ADDR NAME="test2" exec_lxc nerdctl exec vxlan apk add --no-cache iperf3 NAME="test2" exec_lxc nerdctl exec vxlan iperf3 -c $TEST1_VXLAN_ADDR NAME="test" exec_lxc nerdctl rm -f vxlan NAME="test" exec_lxc systemctl --user reset-failed NAME="test2" exec_lxc nerdctl rm -f vxlan ) echo "===== Benchmark: iperf3 client(w/ bypass4netns) server(w/ bypass4netns) with multinode =====" ( NAME="test" exec_lxc systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://$TEST_ADDR:2379 --advertise-client-urls http://$TEST_ADDR:2379 NAME="test" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST_ADDR NAME="test2" exec_lxc systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$TEST_ADDR:2379 --multinode-host-address=$TEST2_ADDR NAME="test" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d -p 8080:5201 --name vxlan $ALPINE_IMAGE sleep infinity" NAME="test" exec_lxc nerdctl exec vxlan apk add --no-cache iperf3 NAME="test" exec_lxc systemd-run --user --unit run-test-iperf3 nerdctl exec vxlan iperf3 -s SERVER_IP=$(NAME="test" exec_lxc nerdctl exec vxlan hostname -i) NAME="test2" exec_lxc /bin/bash -c "sleep 3 && nerdctl run --annotation nerdctl/bypass4netns=true -d --name vxlan $ALPINE_IMAGE sleep infinity" NAME="test2" exec_lxc nerdctl exec vxlan apk add --no-cache iperf3 NAME="test2" exec_lxc nerdctl exec vxlan iperf3 -c $SERVER_IP NAME="test" exec_lxc nerdctl rm -f vxlan NAME="test2" exec_lxc nerdctl rm -f vxlan ) bypass4netns-0.4.1/test/run_test.sh000077500000000000000000000155141460472144300173450ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail source ~/.profile ALPINE_IMAGE="public.ecr.aws/docker/library/alpine:3.16" nerdctl pull --quiet "${ALPINE_IMAGE}" SCRIPT_DIR=$(cd $(dirname $0); pwd) set +u if [ "$1" == "SYNC" ]; then echo "updating source code" rm -rf ~/bypass4netns sudo cp -r /host ~/bypass4netns sudo chown -R ubuntu:ubuntu ~/bypass4netns cd ~/bypass4netns echo "source code is updated" exec $0 "FORK" exit 0 fi cd ~/bypass4netns rm -f bypass4netns bypass4netnsd make sudo make install set -u cd $SCRIPT_DIR set +e systemctl --user stop run-iperf3 systemctl --user reset-failed sleep 1 systemctl --user restart containerd sleep 1 systemctl --user restart buildkit sleep 3 set -e systemd-run --user --unit run-iperf3 iperf3 -s HOST_IP=$(HOST=$(hostname -I); for i in ${HOST[@]}; do echo $i | grep -q "192.168.6."; if [ $? -eq 0 ]; then echo $i; fi; done) ~/bypass4netns/test/seccomp.json.sh | tee /tmp/seccomp.json sudo journalctl --rotate sudo journalctl --vacuum-time=1s echo "===== rootful mode ====" ( set +e sudo nerdctl rm -f test set -ex sudo nerdctl run -d --name test $ALPINE_IMAGE sleep infinity sudo nerdctl exec test apk add --no-cache iperf3 sudo nerdctl exec test iperf3 -c $HOST_IP -t 1 --connect-timeout 1000 # it must success to connect. sudo nerdctl rm -f test ) echo "===== static linked binary test =====" ( set +e systemctl --user stop run-bypass4netns-static nerdctl rm -f test1 nerdctl rm -f test2 systemctl --user reset-failed set -ex IMAGE_NAME="b4ns:static" nerdctl build -f ./DockerfileHttpServer -t $IMAGE_NAME . systemd-run --user --unit run-bypass4netns-static bypass4netns --ignore "127.0.0.0/8,10.0.0.0/8" sleep 1 nerdctl run -d -p 8081:8080 --name test1 $IMAGE_NAME /httpserver -mode server nerdctl run --security-opt seccomp=/tmp/seccomp.json -d --name test2 $IMAGE_NAME sleep infinity nerdctl exec test2 /httpserver -mode client -url http://$HOST_IP:8081/ping nerdctl exec test2 /httpserver -mode client -url http://$HOST_IP:8081/ping nerdctl exec test2 /httpserver -mode client -url http://$HOST_IP:8081/ping COUNT=$(journalctl --user -u run-bypass4netns-static.service | grep 'bypassed connect socket' | wc -l) if [ $COUNT != 3 ]; then echo "static linked binary bypassing not working correctly." exit 1 fi nerdctl rm -f test1 nerdctl rm -f test2 systemctl --user stop run-bypass4netns-static ) echo "===== '--ignore' option test =====" ( set +e systemctl --user stop run-bypass4netns nerdctl rm -f test set -ex systemd-run --user --unit run-bypass4netns bypass4netns --ignore "127.0.0.0/8,10.0.0.0/8,192.168.6.0/24" --debug nerdctl run --security-opt seccomp=/tmp/seccomp.json -d --name test "${ALPINE_IMAGE}" sleep infinity nerdctl exec test apk add --no-cache iperf3 nerdctl exec test iperf3 -c $HOST_IP -t 1 # TODO: this check is dirty. we want better method to check the connect(2) is ignored. journalctl --user -u run-bypass4netns.service | grep "is not bypassed" nerdctl rm -f test systemctl --user stop run-bypass4netns.service ) echo "===== connect(2) test =====" ( systemd-run --user --unit run-bypass4netns bypass4netns --ignore "127.0.0.0/8,10.0.0.0/8" -p 8080:5201 set -x cd $SCRIPT_DIR /bin/bash test_syscalls.sh /tmp/seccomp.json $HOST_IP systemctl --user stop run-bypass4netns.service ) echo "===== Test bypass4netnsd =====" ( set -x source ~/.profile ./test_b4nnd.sh ) echo "===== tracer test (disabled) =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd --handle-c2c-connections=true sleep 1 nerdctl run --annotation nerdctl/bypass4netns=true -d -p 8080:5201 --name test1 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test1 apk add --no-cache iperf3 TEST1_ADDR=$(nerdctl exec test1 hostname -i) systemd-run --user --unit run-test1-iperf3 nerdctl exec test1 iperf3 -s nerdctl network create --subnet "10.4.1.0/24" net-2 nerdctl run --net net-2 --annotation nerdctl/bypass4netns=true -d --name test2 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test2 apk add --no-cache iperf3 nerdctl exec test2 iperf3 -c $TEST1_ADDR -t 1 --connect-timeout 1000 # it must success to connect. nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user stop run-bypass4netnsd ) echo "===== tracer test (enabled) =====" ( set +e systemctl --user stop run-bypass4netnsd nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user reset-failed set -ex systemd-run --user --unit run-bypass4netnsd bypass4netnsd --handle-c2c-connections=true --tracer=true --debug sleep 1 nerdctl run --annotation nerdctl/bypass4netns=true -d -p 8080:5201 --name test1 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test1 apk add --no-cache iperf3 TEST1_ADDR=$(nerdctl exec test1 hostname -i) systemd-run --user --unit run-test1-iperf3 nerdctl exec test1 iperf3 -s nerdctl network create --subnet "10.4.1.0/24" net-2 nerdctl run --net net-2 --annotation nerdctl/bypass4netns=true -d --name test2 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test2 apk add --no-cache iperf3 set +e nerdctl exec test2 iperf3 -c $TEST1_ADDR -t 1 --connect-timeout 1000 # it must not success to connect. if [ $? -eq 0 ]; then echo "tracer seems not working" exit 1 fi set -e nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user stop run-bypass4netnsd ) echo "===== multinode test (single node) ====" ( set +e nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed set -ex systemd-run --user --unit etcd.service /usr/bin/etcd --listen-client-urls http://${HOST_IP}:2379 --advertise-client-urls http://${HOST_IP}:2379 systemd-run --user --unit run-bypass4netnsd bypass4netnsd --multinode=true --multinode-etcd-address=http://$HOST_IP:2379 --multinode-host-address=$HOST_IP --debug sleep 1 nerdctl run --annotation nerdctl/bypass4netns=true -d -p 8080:5201 --name test1 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test1 apk add --no-cache iperf3 TEST1_ADDR=$(nerdctl exec test1 hostname -i) systemd-run --user --unit run-test1-iperf3 nerdctl exec test1 iperf3 -s nerdctl network create --subnet "10.4.1.0/24" net-2 nerdctl run --net net-2 --annotation nerdctl/bypass4netns=true -d --name test2 "${ALPINE_IMAGE}" sleep infinity nerdctl exec test2 apk add --no-cache iperf3 nerdctl exec test2 iperf3 -c $TEST1_ADDR -t 1 --connect-timeout 1000 # it must success to connect. nerdctl rm -f test1 nerdctl rm -f test2 nerdctl network rm net-2 systemctl --user stop run-bypass4netnsd systemctl --user stop etcd.service systemctl --user reset-failed ) bypass4netns-0.4.1/test/seccomp.json.sh000077500000000000000000000013701460472144300200760ustar00rootroot00000000000000#!/bin/sh # Usage: # $ ./seccomp.json.sh >$HOME/seccomp.json # $ nerdctl run -it --rm --security-opt seccomp=$HOME/seccomp.json alpine # TODO: support non-x86 # TODO: inherit the default seccomp profile (https://github.com/containerd/containerd/blob/v1.6.0-rc.1/contrib/seccomp/seccomp_default.go#L52) set -eu cat < /dev/null & nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_connect.py -s -p 8888 --count 2 &> /dev/null & nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_connect.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP --count 2 ## NOTICE ## # currently, bypass4netns supports only TCP. Tests for udp connections are disabled. # test_connect udp #python3 test_connect.py -s -p 8888 -u --count 2 &> /tmp/test_host & #nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_connect.py -s -p 8888 -u --count 2 &> /tmp/test_test2 & #nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_connect.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP -u --count 2 #sleep 5 # check server is not timedout #RESULT=`cat /tmp/test_host /tmp/test_test2` #if [[ "$RESULT" == *timeout* ]]; then # echo "test connect over udp failed" # cat /tmp/test_host # cat /tmp/test_test2 # exit 1 #fi echo "test_connect done." #echo "test_sendto starting..." ## test_sendto tcp #python3 test_sendto.py -s -p 8888 --count 2 &> /dev/null & #nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_sendto.py -s -p 8888 --count 2 &> /dev/null & #nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_sendto.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP --count 2 # ## test_sendto udp #python3 test_sendto.py -s -p 8888 -u --count 2 &> /tmp/test_host & #nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_sendto.py -s -p 8888 -u --count 2 &> /tmp/test_test2 & #nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_sendto.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP -u --count 2 #sleep 5 # ## check server is not timedout #RESULT=`cat /tmp/test_host /tmp/test_test2` #if [[ "$RESULT" == *timeout* ]]; then # echo "test sendto over udp failed" # cat /tmp/test_host # cat /tmp/test_test2 # exit 1 #fi #echo "test_sendto done." # #echo "test_sendmsg starting..." ## test_sendmsg tcp #python3 test_sendmsg.py -s -p 8888 --count 2 &> /dev/null & #nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_sendmsg.py -s -p 8888 --count 2 &> /dev/null & #nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_sendmsg.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP --count 2 # ## test_sendmsg udp #python3 test_sendmsg.py -s -p 8888 -u --count 2 &> /tmp/test_host & #nerdctl exec $TEST_CONTAINER_2 python3 /tmp/test_sendmsg.py -s -p 8888 -u --count 2 &> /tmp/test_test2 & #nerdctl exec $TEST_CONTAINER_1 python3 /tmp/test_sendmsg.py -c -p 8888 --host-ip $HOST_IP --netns-ip $NETNS_IP -u --count 2 #sleep 5 # ## check server is not timedout #RESULT=`cat /tmp/test_host /tmp/test_test2` #if [[ "$RESULT" == *timeout* ]]; then # echo "test sendto over udp failed" # cat /tmp/test_host # cat /tmp/test_test2 # exit 1 #fi #echo "test_sendmsg done." nerdctl rm -f $TEST_CONTAINER_2 nerdctl rm -f $TEST_CONTAINER_1 # rm /tmp/test_host /tmp/test_test2 bypass4netns-0.4.1/test/util.sh000066400000000000000000000004421460472144300164460ustar00rootroot00000000000000#!/bin/bash set -eu -o pipefail function exec_netns() { if [ $EUID -eq 0 ]; then nsenter -t $PID -F -n -- "$@" else nsenter -t $PID -F -U --preserve-credentials -n -- "$@" fi } function exec_lxc() { sudo lxc exec $NAME -- sudo --login --user ubuntu "$@" }