diff --git a/examples/cmd/remote-snapshotter/go.sum b/examples/cmd/remote-snapshotter/go.sum index b11b7edf0..65e3f0802 100644 --- a/examples/cmd/remote-snapshotter/go.sum +++ b/examples/cmd/remote-snapshotter/go.sum @@ -93,6 +93,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -185,6 +186,7 @@ github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -231,6 +233,7 @@ github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= @@ -352,6 +355,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/firecracker-microvm/firecracker-go-sdk v0.22.1-0.20220427214706-47505a9cf951 h1:j1zRfar/9U22TeptQIwXB07hyuctgb6++HIxP0nXBhI= github.com/firecracker-microvm/firecracker-go-sdk v0.22.1-0.20220427214706-47505a9cf951/go.mod h1:60W3x6ftClUbRKpqXl7XvrhM/Uv3tochNRq+RlZsd1M= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -382,6 +386,7 @@ github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= @@ -562,6 +567,7 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -570,6 +576,7 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -633,10 +640,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -661,9 +670,12 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mdlayher/socket v0.2.0 h1:EY4YQd6hTAg2tcXF84p5DTHazShE50u5HeBzBaNgjkA= github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/mdlayher/vsock v1.1.1 h1:8lFuiXQnmICBrCIIA9PMgVSke6Fg6V4+r0v7r55k88I= github.com/mdlayher/vsock v1.1.1/go.mod h1:Y43jzcy7KM3QB+/FK15pfqGxDMCMzUXWegEfIbSM18U= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.25 h1:dFwPR6SfLtrSwgDcIq2bcU/gVutB4sNApq2HBdqcakg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -848,7 +860,9 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -990,6 +1004,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1014,6 +1029,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1480,6 +1496,7 @@ gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/examples/cmd/remote-snapshotter/remote_snapshotter.go b/examples/cmd/remote-snapshotter/remote_snapshotter.go index 0984f1834..123a4e156 100644 --- a/examples/cmd/remote-snapshotter/remote_snapshotter.go +++ b/examples/cmd/remote-snapshotter/remote_snapshotter.go @@ -26,6 +26,7 @@ import ( fcclient "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" ) const ( @@ -92,7 +93,7 @@ func main() { defer fcClient.Close() fmt.Println("Creating VM") - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, NetworkInterfaces: []*proto.FirecrackerNetworkInterface{{ AllowMMDS: true, @@ -121,10 +122,14 @@ func main() { fmt.Println("Pulling the image") image, err := client.Pull(ctx, imageRef, - containerd.WithPullSnapshotter(snapshotter), + containerd.WithPullSnapshotter(snapshotter, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), containerd.WithPullUnpack, // stargz labels to tell the snapshotter to lazily load the image - containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(imageRef, 10*1024*1024))) + containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(imageRef, 10*1024*1024)), + ) if err != nil { fmt.Println(err) return diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index e1ee2970f..6bbe15ad2 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -207,6 +207,7 @@ type CreateVMResponse struct { LogFifoPath string `protobuf:"bytes,3,opt,name=LogFifoPath,json=logFifoPath,proto3" json:"LogFifoPath,omitempty"` MetricsFifoPath string `protobuf:"bytes,4,opt,name=MetricsFifoPath,json=metricsFifoPath,proto3" json:"MetricsFifoPath,omitempty"` CgroupPath string `protobuf:"bytes,5,opt,name=CgroupPath,json=cgroupPath,proto3" json:"CgroupPath,omitempty"` + VSockPath string `protobuf:"bytes,6,opt,name=VSockPath,json=vSockPath,proto3" json:"VSockPath,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -271,6 +272,13 @@ func (m *CreateVMResponse) GetCgroupPath() string { return "" } +func (m *CreateVMResponse) GetVSockPath() string { + if m != nil { + return m.VSockPath + } + return "" +} + type PauseVMRequest struct { VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1174,71 +1182,71 @@ func init() { func init() { proto.RegisterFile("firecracker.proto", fileDescriptor_a73317e9fb8da571) } var fileDescriptor_a73317e9fb8da571 = []byte{ - // 1044 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0x4f, 0x6f, 0xe2, 0x46, - 0x14, 0xaf, 0x63, 0x20, 0xe1, 0x11, 0x48, 0x18, 0x25, 0x59, 0xef, 0x2a, 0x8a, 0x90, 0xd5, 0x6e, - 0xd1, 0xaa, 0x8d, 0xd4, 0xa4, 0x87, 0xaa, 0x97, 0x2e, 0x81, 0x25, 0xcb, 0x6e, 0x9d, 0x45, 0x43, - 0x12, 0xa9, 0xed, 0x69, 0x62, 0x1e, 0xc4, 0x8b, 0xed, 0xa1, 0x9e, 0x31, 0xbb, 0xf9, 0x5a, 0x3d, - 0xf5, 0x6b, 0xb4, 0xdf, 0xa5, 0xf7, 0x6a, 0xc6, 0x06, 0x1b, 0x92, 0xb2, 0x91, 0x7a, 0xda, 0x13, - 0xcc, 0xef, 0xfd, 0x66, 0xde, 0x9f, 0xdf, 0x9b, 0x37, 0x86, 0xfa, 0xc8, 0x8b, 0xd0, 0x8d, 0x98, - 0x3b, 0xc1, 0xe8, 0x78, 0x1a, 0x71, 0xc9, 0x9f, 0x55, 0xe4, 0xdd, 0x14, 0x45, 0xb2, 0xb0, 0xff, - 0x2c, 0xc2, 0x4e, 0x3b, 0x42, 0x26, 0xf1, 0xda, 0xa1, 0xf8, 0x7b, 0x8c, 0x42, 0x12, 0x02, 0x85, - 0x6b, 0xa7, 0xd7, 0xb1, 0x8c, 0x86, 0xd1, 0x2c, 0xd3, 0xc2, 0xcc, 0xe9, 0x75, 0xc8, 0x4b, 0x00, - 0x87, 0xb9, 0xb7, 0x5e, 0x88, 0xed, 0xd1, 0xd8, 0xda, 0x68, 0x18, 0xcd, 0xca, 0x49, 0xe3, 0xb8, - 0x9b, 0x1d, 0x3e, 0xb7, 0xf2, 0x70, 0xe4, 0x8d, 0xe3, 0x88, 0x49, 0x8f, 0x87, 0x14, 0x82, 0xc5, - 0x1e, 0xd2, 0x84, 0x9d, 0xb7, 0x18, 0x85, 0xe8, 0xf7, 0x02, 0x36, 0xc6, 0x3e, 0x93, 0xb7, 0x96, - 0xa9, 0x1d, 0xec, 0x4c, 0x96, 0x61, 0x72, 0x04, 0x90, 0x30, 0x5b, 0xd1, 0x58, 0x58, 0x05, 0x4d, - 0x82, 0xc9, 0x02, 0x21, 0xa7, 0x50, 0xa6, 0x9c, 0xcb, 0x4e, 0xe4, 0xcd, 0xd0, 0x2a, 0xea, 0x50, - 0xf6, 0xf3, 0xa1, 0x2c, 0x8c, 0xb4, 0x1c, 0xcd, 0xff, 0x92, 0x1f, 0xa0, 0xa2, 0xff, 0x38, 0x3c, - 0x0e, 0xa5, 0xb0, 0x4a, 0x0d, 0xb3, 0x59, 0x39, 0x39, 0xc8, 0x6f, 0xcb, 0xcc, 0xb4, 0x32, 0xcc, - 0xa8, 0xe4, 0x0d, 0xd4, 0x2f, 0x50, 0x7e, 0xe0, 0xd1, 0xa4, 0x17, 0x4a, 0x8c, 0x46, 0xcc, 0x45, - 0x61, 0x6d, 0xea, 0xfd, 0x87, 0xf9, 0xfd, 0xab, 0x24, 0x5a, 0x0f, 0x57, 0xb7, 0x91, 0xe7, 0x50, - 0x6b, 0xf3, 0x50, 0x32, 0x2f, 0xc4, 0xa8, 0xad, 0x8e, 0xb7, 0xb6, 0x1a, 0x46, 0xb3, 0x48, 0x6b, - 0xee, 0x12, 0x4a, 0x7e, 0x04, 0xeb, 0xd5, 0x47, 0x4f, 0xb6, 0x46, 0x12, 0xa3, 0x96, 0xef, 0x5f, - 0x32, 0x31, 0x11, 0x1d, 0xf4, 0x51, 0xe2, 0xd0, 0x2a, 0x37, 0x8c, 0xe6, 0x16, 0xb5, 0xf0, 0x3f, - 0xec, 0xe4, 0x3b, 0xd8, 0x7e, 0xc3, 0x3c, 0x5f, 0x1d, 0xa5, 0xb4, 0xb0, 0x40, 0x57, 0xa8, 0x7a, - 0x9c, 0x07, 0xe9, 0xf6, 0xfb, 0xdc, 0x4a, 0x85, 0x75, 0xe9, 0x05, 0xc8, 0x63, 0x39, 0x40, 0x97, - 0x87, 0x43, 0x61, 0x55, 0x1a, 0x46, 0xb3, 0x4a, 0x6b, 0x72, 0x09, 0x25, 0x0d, 0xa8, 0xfc, 0xcc, - 0xc7, 0x5d, 0x6f, 0xc4, 0xb5, 0x7e, 0xdb, 0x5a, 0x9a, 0x8a, 0x9f, 0x41, 0x4a, 0x65, 0x07, 0x65, - 0xe4, 0xb9, 0x62, 0xc1, 0xaa, 0x26, 0x2a, 0x07, 0xcb, 0x30, 0xf9, 0x09, 0xaa, 0x67, 0xcc, 0xf7, - 0x39, 0x0f, 0x3b, 0x38, 0xf3, 0x5c, 0xb4, 0x6a, 0x3a, 0xce, 0xa7, 0xf9, 0x92, 0x2e, 0x11, 0x68, - 0xf5, 0x26, 0xbf, 0xb4, 0xff, 0x30, 0x60, 0x37, 0x6b, 0x5d, 0x31, 0xe5, 0xa1, 0xc0, 0x07, 0x7b, - 0xf7, 0x08, 0x60, 0xc0, 0xdd, 0x09, 0x4a, 0x1d, 0xce, 0x46, 0xd2, 0x4f, 0x62, 0x81, 0xac, 0x66, - 0x65, 0x3e, 0x2a, 0xab, 0xc2, 0xc3, 0x59, 0x1d, 0x01, 0xb4, 0xc7, 0x11, 0x8f, 0xa7, 0x9a, 0x54, - 0x4c, 0x7c, 0xb9, 0x0b, 0xc4, 0xfe, 0x12, 0x6a, 0x7d, 0x16, 0x8b, 0xf5, 0xb7, 0xcd, 0xfe, 0x0a, - 0x76, 0x28, 0x8a, 0x38, 0xf8, 0x04, 0xed, 0x2d, 0x54, 0x07, 0x92, 0x4f, 0xd7, 0xdf, 0xdc, 0xfb, - 0xda, 0x6e, 0x3c, 0xa4, 0xad, 0xfd, 0x1c, 0x76, 0xcf, 0x51, 0x5e, 0x3b, 0xbd, 0x70, 0xc4, 0xd7, - 0x39, 0xfd, 0xcb, 0x80, 0x7a, 0x8e, 0xf8, 0x79, 0xd4, 0x9d, 0x1c, 0x42, 0xf9, 0x5a, 0x05, 0xa3, - 0xcd, 0x25, 0x6d, 0x2e, 0xcf, 0xe6, 0x80, 0xdd, 0x85, 0xbd, 0x81, 0x4a, 0xc9, 0x41, 0xc9, 0x86, - 0x4c, 0xb2, 0x75, 0xf5, 0x7c, 0x06, 0x5b, 0x73, 0x5a, 0x9a, 0xd3, 0x56, 0x90, 0xae, 0xed, 0x1e, - 0x3c, 0xb9, 0x9a, 0x0e, 0x75, 0x47, 0xfe, 0xdf, 0xa3, 0x5e, 0xc0, 0xde, 0xf9, 0x23, 0x43, 0xb2, - 0x4f, 0x61, 0x7f, 0x85, 0x9b, 0xaa, 0x92, 0x77, 0x60, 0xac, 0x38, 0xf8, 0xdb, 0x58, 0x9e, 0x13, - 0x64, 0x0f, 0x8a, 0x17, 0x28, 0x2f, 0x06, 0x29, 0xb3, 0x18, 0xaa, 0x85, 0xf2, 0xd7, 0xee, 0x5f, - 0x89, 0x34, 0xbe, 0x82, 0xdb, 0xbf, 0x12, 0x0a, 0x73, 0x30, 0x10, 0xa9, 0x62, 0x85, 0x00, 0x03, - 0x41, 0x76, 0xc1, 0xbc, 0xea, 0x75, 0xb4, 0x3c, 0x55, 0x6a, 0xc6, 0xbd, 0x8e, 0x42, 0xce, 0x7b, - 0x1d, 0xad, 0x45, 0x95, 0x9a, 0xe3, 0xa4, 0x21, 0x72, 0x22, 0x95, 0xee, 0x89, 0xf4, 0x12, 0xea, - 0x7a, 0x08, 0xbf, 0xfa, 0x38, 0xe5, 0x02, 0xfb, 0xdc, 0xf7, 0xdc, 0x3b, 0x6b, 0xb3, 0x61, 0x34, - 0x6b, 0x27, 0xe4, 0xf8, 0x9e, 0x85, 0xd6, 0x87, 0xab, 0x90, 0xfd, 0x1a, 0xf6, 0x12, 0x01, 0xd2, - 0xc9, 0xb1, 0xae, 0xfa, 0x87, 0x50, 0x6e, 0x05, 0x6a, 0xda, 0x3a, 0xde, 0x8d, 0x4e, 0xcf, 0xa4, - 0x65, 0x36, 0x07, 0xec, 0x6f, 0xe1, 0xc9, 0x39, 0xca, 0xf4, 0x98, 0x74, 0x68, 0xae, 0x91, 0xe0, - 0x37, 0xb0, 0xee, 0xd3, 0x53, 0x15, 0xb2, 0x49, 0x97, 0x4e, 0x64, 0xe3, 0xb1, 0x93, 0x2e, 0xe1, - 0xdb, 0xdf, 0xc0, 0x41, 0x76, 0xf8, 0x40, 0x32, 0x29, 0xd6, 0x85, 0xf2, 0x8f, 0x99, 0x0f, 0x3d, - 0xa5, 0xa7, 0xa1, 0xa8, 0x9c, 0x5d, 0x19, 0x33, 0x5f, 0xe5, 0x6c, 0xa4, 0x39, 0xcf, 0x01, 0x75, - 0x21, 0x13, 0x6b, 0x9f, 0x8d, 0x51, 0xa4, 0x35, 0xa9, 0xb0, 0x0c, 0x52, 0x17, 0xb2, 0x35, 0x63, - 0x9e, 0xcf, 0x6e, 0x7c, 0x74, 0x30, 0xe0, 0xd1, 0x9d, 0x6e, 0x02, 0x93, 0xee, 0xb0, 0x65, 0x58, - 0x69, 0xdd, 0xf1, 0xc4, 0xa4, 0xcd, 0xdc, 0x5b, 0x4c, 0x1e, 0x71, 0x93, 0xc2, 0x70, 0x81, 0x28, - 0x7b, 0x37, 0xc2, 0xf9, 0x21, 0xc5, 0xc4, 0x3e, 0x5a, 0x20, 0xe4, 0x18, 0xc8, 0xeb, 0x78, 0x8c, - 0xd2, 0xbf, 0x69, 0xf9, 0x3e, 0x77, 0xf5, 0xf7, 0x84, 0xd0, 0x3d, 0x63, 0x52, 0x72, 0x7b, 0xcf, - 0xa2, 0x22, 0x4b, 0xf9, 0x5d, 0xe6, 0xf9, 0x71, 0xa4, 0xdf, 0x68, 0x1d, 0xd9, 0xed, 0x32, 0xac, - 0xb2, 0x74, 0xd8, 0x7b, 0x1e, 0x75, 0x59, 0xec, 0x4b, 0xa1, 0x1f, 0x60, 0x93, 0x56, 0x82, 0x0c, - 0xd2, 0x0c, 0x2f, 0x5c, 0x30, 0xca, 0x29, 0x23, 0x83, 0xc8, 0x01, 0x94, 0x06, 0x1f, 0xd8, 0xb4, - 0x17, 0xea, 0xd7, 0xd5, 0xa4, 0x25, 0xa1, 0x57, 0xc4, 0x82, 0x4d, 0x85, 0xbf, 0x8b, 0xa5, 0x7e, - 0x41, 0x4d, 0xba, 0x29, 0x92, 0xa5, 0xaa, 0xfc, 0x25, 0x8b, 0xc6, 0xa8, 0xbb, 0x6d, 0x3b, 0xa9, - 0xbc, 0x9c, 0x03, 0xca, 0x63, 0x62, 0x4d, 0x2a, 0x5f, 0x4d, 0x3c, 0xca, 0x0c, 0xd2, 0x0c, 0x2e, - 0x99, 0x9f, 0x16, 0xac, 0x96, 0x32, 0x32, 0xc8, 0x46, 0x78, 0xba, 0xd4, 0xfb, 0x9f, 0x6a, 0x14, - 0xf2, 0x3d, 0xec, 0x6b, 0x4e, 0x9f, 0xfb, 0xbe, 0x17, 0x8e, 0xf5, 0x67, 0xca, 0x8c, 0xf9, 0x73, - 0xe1, 0xf7, 0xc5, 0x43, 0xc6, 0x17, 0x5f, 0x3f, 0x70, 0x49, 0xc9, 0x16, 0x14, 0xda, 0xef, 0xfa, - 0xbf, 0xec, 0x7e, 0xa1, 0xfe, 0x9d, 0xf5, 0x2e, 0x3a, 0xbb, 0xc6, 0xd9, 0xe6, 0xaf, 0x45, 0xfd, - 0x8d, 0x79, 0x53, 0xd2, 0x3f, 0xa7, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x58, 0x5c, 0x22, 0x3c, - 0x8c, 0x0a, 0x00, 0x00, + // 1042 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x56, 0xdd, 0x6e, 0xe2, 0x46, + 0x14, 0xae, 0x63, 0x20, 0xe1, 0x10, 0x48, 0x18, 0x25, 0x59, 0xef, 0x2a, 0x8a, 0x10, 0x6a, 0xb7, + 0x68, 0xd5, 0x46, 0x6a, 0xd2, 0x8b, 0xaa, 0x37, 0x5d, 0x02, 0x4b, 0x96, 0xdd, 0x3a, 0x8b, 0x86, + 0x24, 0x52, 0xdb, 0xab, 0x89, 0x39, 0x38, 0x5e, 0x6c, 0x0f, 0xf5, 0x8c, 0xd9, 0xcd, 0x9b, 0xf5, + 0x31, 0xda, 0xbe, 0x4b, 0xef, 0xab, 0x19, 0x1b, 0x6c, 0x48, 0xca, 0x46, 0xea, 0xdd, 0x5e, 0xc1, + 0x7c, 0xe7, 0x9b, 0x39, 0x3f, 0xdf, 0x99, 0x33, 0x86, 0xfa, 0xd8, 0x8b, 0xd0, 0x89, 0x98, 0x33, + 0xc1, 0xe8, 0x78, 0x1a, 0x71, 0xc9, 0x9f, 0x55, 0xe4, 0xdd, 0x14, 0x45, 0xb2, 0x68, 0xfe, 0x51, + 0x84, 0x9d, 0x4e, 0x84, 0x4c, 0xe2, 0xb5, 0x4d, 0xf1, 0xf7, 0x18, 0x85, 0x24, 0x04, 0x0a, 0xd7, + 0x76, 0xbf, 0x6b, 0x19, 0x0d, 0xa3, 0x55, 0xa6, 0x85, 0x99, 0xdd, 0xef, 0x92, 0x97, 0x00, 0x36, + 0x73, 0x6e, 0xbd, 0x10, 0x3b, 0x63, 0xd7, 0xda, 0x68, 0x18, 0xad, 0xca, 0x49, 0xe3, 0xb8, 0x97, + 0x1d, 0x3e, 0xb7, 0xf2, 0x70, 0xec, 0xb9, 0x71, 0xc4, 0xa4, 0xc7, 0x43, 0x0a, 0xc1, 0x62, 0x0f, + 0x69, 0xc1, 0xce, 0x5b, 0x8c, 0x42, 0xf4, 0xfb, 0x01, 0x73, 0x71, 0xc0, 0xe4, 0xad, 0x65, 0x6a, + 0x07, 0x3b, 0x93, 0x65, 0x98, 0x1c, 0x01, 0x24, 0xcc, 0x76, 0xe4, 0x0a, 0xab, 0xa0, 0x49, 0x30, + 0x59, 0x20, 0xe4, 0x14, 0xca, 0x94, 0x73, 0xd9, 0x8d, 0xbc, 0x19, 0x5a, 0x45, 0x1d, 0xca, 0x7e, + 0x3e, 0x94, 0x85, 0x91, 0x96, 0xa3, 0xf9, 0x5f, 0xf2, 0x03, 0x54, 0xf4, 0x1f, 0x9b, 0xc7, 0xa1, + 0x14, 0x56, 0xa9, 0x61, 0xb6, 0x2a, 0x27, 0x07, 0xf9, 0x6d, 0x99, 0x99, 0x56, 0x46, 0x19, 0x95, + 0xbc, 0x81, 0xfa, 0x05, 0xca, 0x0f, 0x3c, 0x9a, 0xf4, 0x43, 0x89, 0xd1, 0x98, 0x39, 0x28, 0xac, + 0x4d, 0xbd, 0xff, 0x30, 0xbf, 0x7f, 0x95, 0x44, 0xeb, 0xe1, 0xea, 0x36, 0xf2, 0x1c, 0x6a, 0x1d, + 0x1e, 0x4a, 0xe6, 0x85, 0x18, 0x75, 0xd4, 0xf1, 0xd6, 0x56, 0xc3, 0x68, 0x15, 0x69, 0xcd, 0x59, + 0x42, 0xc9, 0x8f, 0x60, 0xbd, 0xfa, 0xe8, 0xc9, 0xf6, 0x58, 0x62, 0xd4, 0xf6, 0xfd, 0x4b, 0x26, + 0x26, 0xa2, 0x8b, 0x3e, 0x4a, 0x1c, 0x59, 0xe5, 0x86, 0xd1, 0xda, 0xa2, 0x16, 0xfe, 0x87, 0x9d, + 0x7c, 0x07, 0xdb, 0x6f, 0x98, 0xe7, 0xab, 0xa3, 0x94, 0x16, 0x16, 0xe8, 0x0a, 0x55, 0x8f, 0xf3, + 0x20, 0xdd, 0x7e, 0x9f, 0x5b, 0xa9, 0xb0, 0x2e, 0xbd, 0x00, 0x79, 0x2c, 0x87, 0xe8, 0xf0, 0x70, + 0x24, 0xac, 0x4a, 0xc3, 0x68, 0x55, 0x69, 0x4d, 0x2e, 0xa1, 0xa4, 0x01, 0x95, 0x9f, 0xb9, 0xdb, + 0xf3, 0xc6, 0x5c, 0xeb, 0xb7, 0xad, 0xa5, 0xa9, 0xf8, 0x19, 0xa4, 0x54, 0xb6, 0x51, 0x46, 0x9e, + 0x23, 0x16, 0xac, 0x6a, 0xa2, 0x72, 0xb0, 0x0c, 0x93, 0x9f, 0xa0, 0x7a, 0xc6, 0x7c, 0x9f, 0xf3, + 0xb0, 0x8b, 0x33, 0xcf, 0x41, 0xab, 0xa6, 0xe3, 0x7c, 0x9a, 0x2f, 0xe9, 0x12, 0x81, 0x56, 0x6f, + 0xf2, 0xcb, 0xe6, 0x9f, 0x06, 0xec, 0x66, 0xad, 0x2b, 0xa6, 0x3c, 0x14, 0xf8, 0x60, 0xef, 0x1e, + 0x01, 0x0c, 0xb9, 0x33, 0x41, 0xa9, 0xc3, 0xd9, 0x48, 0xfa, 0x49, 0x2c, 0x90, 0xd5, 0xac, 0xcc, + 0x47, 0x65, 0x55, 0x78, 0x38, 0xab, 0x23, 0x80, 0x8e, 0x1b, 0xf1, 0x78, 0xaa, 0x49, 0xc5, 0xc4, + 0x97, 0xb3, 0x40, 0xc8, 0x21, 0x94, 0xaf, 0x55, 0x30, 0xda, 0x5c, 0xd2, 0xe6, 0xf2, 0x6c, 0x0e, + 0x34, 0xbf, 0x84, 0xda, 0x80, 0xc5, 0x62, 0xfd, 0x5d, 0x6c, 0x7e, 0x05, 0x3b, 0x14, 0x45, 0x1c, + 0x7c, 0x82, 0xf6, 0x16, 0xaa, 0x43, 0xc9, 0xa7, 0xeb, 0xef, 0xf5, 0x7d, 0xe5, 0x37, 0x1e, 0x52, + 0xbe, 0xf9, 0x1c, 0x76, 0xcf, 0x51, 0x5e, 0xdb, 0xfd, 0x70, 0xcc, 0xd7, 0x39, 0xfd, 0xcb, 0x80, + 0x7a, 0x8e, 0xf8, 0x59, 0xa8, 0xd2, 0x83, 0xbd, 0xa1, 0x4a, 0xc9, 0x46, 0xc9, 0x46, 0x4c, 0xb2, + 0x75, 0xf5, 0x7c, 0x06, 0x5b, 0x73, 0x5a, 0x9a, 0xd3, 0x56, 0x90, 0xae, 0x9b, 0x7d, 0x78, 0x72, + 0x35, 0x1d, 0xe9, 0x7e, 0xfd, 0xbf, 0x47, 0xbd, 0x80, 0xbd, 0xf3, 0x47, 0x86, 0xd4, 0x3c, 0x85, + 0xfd, 0x15, 0x6e, 0xaa, 0x4a, 0xde, 0x81, 0xb1, 0xe2, 0xe0, 0x6f, 0x63, 0x79, 0x8a, 0x90, 0x3d, + 0x28, 0x5e, 0xa0, 0xbc, 0x18, 0xa6, 0xcc, 0x62, 0xa8, 0x16, 0xca, 0x5f, 0x67, 0x70, 0x25, 0xd2, + 0xf8, 0x0a, 0xce, 0xe0, 0x4a, 0x28, 0xcc, 0xc6, 0x40, 0xa4, 0x8a, 0x15, 0x02, 0x0c, 0x04, 0xd9, + 0x05, 0xf3, 0xaa, 0xdf, 0xd5, 0xf2, 0x54, 0xa9, 0x19, 0xf7, 0xbb, 0x0a, 0x39, 0xef, 0x77, 0xb5, + 0x16, 0x55, 0x6a, 0xba, 0x49, 0x43, 0xe4, 0x44, 0x2a, 0xdd, 0x13, 0xe9, 0x25, 0xd4, 0xf5, 0x88, + 0x7e, 0xf5, 0x71, 0xca, 0x05, 0x0e, 0xb8, 0xef, 0x39, 0x77, 0xd6, 0x66, 0xc3, 0x68, 0xd5, 0x4e, + 0xc8, 0xf1, 0x3d, 0x0b, 0xad, 0x8f, 0x56, 0xa1, 0xe6, 0x6b, 0xd8, 0x4b, 0x04, 0x48, 0xe7, 0xca, + 0xba, 0xea, 0x1f, 0x42, 0xb9, 0x1d, 0xa8, 0x59, 0x6c, 0x7b, 0x37, 0x3a, 0x3d, 0x93, 0x96, 0xd9, + 0x1c, 0x68, 0x7e, 0x0b, 0x4f, 0xce, 0x51, 0xa6, 0xc7, 0xa4, 0x23, 0x75, 0x8d, 0x04, 0xbf, 0x81, + 0x75, 0x9f, 0x9e, 0xaa, 0x90, 0xcd, 0xc1, 0x74, 0x5e, 0x1b, 0x8f, 0x9d, 0x83, 0x09, 0xbf, 0xf9, + 0x0d, 0x1c, 0x64, 0x87, 0x0f, 0x25, 0x93, 0x62, 0x5d, 0x28, 0xff, 0x98, 0xf9, 0xd0, 0x53, 0x7a, + 0x1a, 0x8a, 0xca, 0xd9, 0x91, 0x31, 0xf3, 0x55, 0xce, 0x46, 0x9a, 0xf3, 0x1c, 0x50, 0x17, 0x32, + 0xb1, 0x0e, 0x98, 0x8b, 0x22, 0xad, 0x49, 0x85, 0x65, 0x90, 0xba, 0x90, 0xed, 0x19, 0xf3, 0x7c, + 0x76, 0xe3, 0xa3, 0x8d, 0x01, 0x8f, 0xee, 0x74, 0x13, 0x98, 0x74, 0x87, 0x2d, 0xc3, 0x4a, 0xeb, + 0xae, 0x27, 0x26, 0x1d, 0xe6, 0xdc, 0x62, 0xf2, 0xc4, 0x9b, 0x14, 0x46, 0x0b, 0x44, 0xd9, 0x7b, + 0x11, 0xce, 0x0f, 0x29, 0x26, 0xf6, 0xf1, 0x02, 0x21, 0xc7, 0x40, 0x5e, 0xc7, 0x2e, 0x4a, 0xff, + 0xa6, 0xed, 0xfb, 0xdc, 0xd1, 0x5f, 0x1b, 0x42, 0xf7, 0x8c, 0x49, 0xc9, 0xed, 0x3d, 0x8b, 0x8a, + 0x2c, 0xe5, 0xf7, 0x98, 0xe7, 0xc7, 0x91, 0x7e, 0xc1, 0x75, 0x64, 0xb7, 0xcb, 0xb0, 0xca, 0xd2, + 0x66, 0xef, 0x79, 0xd4, 0x63, 0xb1, 0x2f, 0x85, 0x7e, 0x9e, 0x4d, 0x5a, 0x09, 0x32, 0x48, 0x33, + 0xbc, 0x70, 0xc1, 0x28, 0xa7, 0x8c, 0x0c, 0x22, 0x07, 0x50, 0x1a, 0x7e, 0x60, 0xd3, 0x7e, 0xa8, + 0xdf, 0x5e, 0x93, 0x96, 0x84, 0x5e, 0x11, 0x0b, 0x36, 0x15, 0xfe, 0x2e, 0x96, 0xfa, 0x7d, 0x35, + 0xe9, 0xa6, 0x48, 0x96, 0xaa, 0xf2, 0x97, 0x2c, 0x72, 0x51, 0x77, 0xdb, 0x76, 0x52, 0x79, 0x39, + 0x07, 0x94, 0xc7, 0xc4, 0x9a, 0x54, 0xbe, 0x9a, 0x78, 0x94, 0x19, 0xa4, 0x19, 0x5c, 0x32, 0x3f, + 0x2d, 0x58, 0x2d, 0x65, 0x64, 0x50, 0x13, 0xe1, 0xe9, 0x52, 0xef, 0x7f, 0xaa, 0x51, 0xc8, 0xf7, + 0xb0, 0xaf, 0x39, 0x03, 0xee, 0xfb, 0x5e, 0xe8, 0xea, 0x8f, 0x98, 0x19, 0xf3, 0xe7, 0xc2, 0xef, + 0x8b, 0x87, 0x8c, 0x2f, 0xbe, 0x7e, 0xe0, 0x92, 0x92, 0x2d, 0x28, 0x74, 0xde, 0x0d, 0x7e, 0xd9, + 0xfd, 0x42, 0xfd, 0x3b, 0xeb, 0x5f, 0x74, 0x77, 0x8d, 0xb3, 0xcd, 0x5f, 0x8b, 0xfa, 0x0b, 0xf4, + 0xa6, 0xa4, 0x7f, 0x4e, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xd1, 0x2a, 0x11, 0x46, 0xaa, 0x0a, + 0x00, 0x00, } diff --git a/proto/firecracker.proto b/proto/firecracker.proto index 3dfe44db9..86f308bc7 100644 --- a/proto/firecracker.proto +++ b/proto/firecracker.proto @@ -49,6 +49,7 @@ message CreateVMResponse { string LogFifoPath = 3; string MetricsFifoPath = 4; string CgroupPath = 5; + string VSockPath = 6; } message PauseVMRequest { diff --git a/runtime/service.go b/runtime/service.go index 6eb5ccb3c..fdeebaad8 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -502,6 +502,7 @@ func (s *service) CreateVM(requestCtx context.Context, request *proto.CreateVMRe resp.MetricsFifoPath = s.machineConfig.MetricsFifo resp.LogFifoPath = s.machineConfig.LogFifo resp.SocketPath = s.shimDir.FirecrackerSockPath() + resp.VSockPath = s.shimDir.FirecrackerVSockPath() if c, ok := s.jailer.(cgroupPather); ok { resp.CgroupPath = c.CgroupPath() } diff --git a/snapshotter/Makefile b/snapshotter/Makefile index bfc663aae..118e4abed 100644 --- a/snapshotter/Makefile +++ b/snapshotter/Makefile @@ -34,15 +34,12 @@ INTEG_TESTNAMES=$(shell docker run --rm \ $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ -c "go test -list . | sed '$$d' | grep $(INTEG_TEST_SUFFIX)") -all: demux-snapshotter http-address-resolver +all: demux-snapshotter demux-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM) go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ -http-address-resolver: $(SOURCES) $(GOMOD) $(GOSUM) - go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ internal/http_address_resolver.go - -install: demux-snapshotter http-address-resolver +install: demux-snapshotter install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $^ test: @@ -79,6 +76,8 @@ logs: clean: - rm -f demux-snapshotter +# Leaving legacy binary cleanup so that existing copies can get to a clean +# state after pulling from main - rm -f http-address-resolver distclean: clean diff --git a/snapshotter/app/service.go b/snapshotter/app/service.go index 101b2d2fe..b654daf80 100644 --- a/snapshotter/app/service.go +++ b/snapshotter/app/service.go @@ -37,7 +37,6 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/metrics" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/metrics/discovery" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" - proxyaddress "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy/address" ) // Run the demultiplexing snapshotter service. @@ -137,25 +136,7 @@ func Run(config config.Config) error { return nil } -func initResolver(config config.Config) (proxyaddress.Resolver, error) { - resolverConfig := config.Snapshotter.Proxy.Address.Resolver - switch resolverConfig.Type { - case "http": - return proxyaddress.NewHTTPResolver(resolverConfig.Address), nil - default: - return nil, fmt.Errorf("invalid resolver type: %s", resolverConfig.Type) - } -} - -const base10 = 10 -const bits32 = 32 - func initCache(config config.Config, monitor *metrics.Monitor) (*cache.RemoteSnapshotterCache, error) { - resolver, err := initResolver(config) - if err != nil { - return nil, err - } - dialTimeout, err := time.ParseDuration(config.Snapshotter.Dialer.Timeout) if err != nil { return nil, fmt.Errorf("Error parsing dialer retry interval from config: %w", err) @@ -172,48 +153,31 @@ func initCache(config config.Config, monitor *metrics.Monitor) (*cache.RemoteSna ) } - dial := func(ctx context.Context, namespace string) (net.Conn, error) { - r := resolver - response, err := r.Get(namespace) - - if err != nil { - return nil, err - } - host := response.Address - port, err := strconv.ParseUint(response.SnapshotterPort, base10, bits32) - if err != nil { - return nil, err - } - + dial := func(ctx context.Context, host string) (net.Conn, error) { + // Todo: wire RemoteSnapshotterConfig here somehow + port := uint64(10000) return vsockDial(ctx, host, port) } dialer := proxy.Dialer{Dial: dial, Timeout: dialTimeout} - fetch := func(ctx context.Context, namespace string) (*proxy.RemoteSnapshotter, error) { - r := resolver - response, err := r.Get(namespace) - if err != nil { - return nil, err - } - host := response.Address - port, err := strconv.ParseUint(response.SnapshotterPort, base10, bits32) - if err != nil { - return nil, err - } + fetch := func(ctx context.Context, snapshotterConfig proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { dial := func(ctx context.Context, namespace string) (net.Conn, error) { - return vsockDial(ctx, host, port) + return vsockDial(ctx, snapshotterConfig.VSockPath, uint64(snapshotterConfig.RemoteSnapshotterPort)) } var metricsProxy *metrics.Proxy if config.Snapshotter.Metrics.Enable { - metricsProxy, err = initMetricsProxy(config, monitor, host, response.MetricsPort, response.Labels) + metricsProxy, err = initMetricsProxy(config, monitor, + snapshotterConfig.VSockPath, + snapshotterConfig.MetricsPort, + snapshotterConfig.MetricsLabels) if err != nil { return nil, err } } - return proxy.NewRemoteSnapshotter(ctx, host, dial, metricsProxy) + return proxy.NewRemoteSnapshotter(ctx, snapshotterConfig.VSockPath, dial, metricsProxy) } opts := make([]cache.SnapshotterCacheOption, 0) @@ -247,14 +211,9 @@ func initMetricsProxyMonitor(portRange string) (*metrics.Monitor, error) { return metrics.NewMonitor(lower, upper) } -func initMetricsProxy(config config.Config, monitor *metrics.Monitor, host, port string, labels map[string]string) (*metrics.Proxy, error) { - metricsPort, err := strconv.ParseUint(port, base10, bits32) - if err != nil { - return nil, err - } - +func initMetricsProxy(config config.Config, monitor *metrics.Monitor, host string, port uint32, labels map[string]string) (*metrics.Proxy, error) { metricsDialer := func(ctx context.Context, _, _ string) (net.Conn, error) { - return vsock.DialContext(ctx, host, uint32(metricsPort), vsock.WithLogger(log.G(ctx))) + return vsock.DialContext(ctx, host, port, vsock.WithLogger(log.G(ctx))) } metricsHost := config.Snapshotter.Metrics.Host diff --git a/snapshotter/demux/cache/cache.go b/snapshotter/demux/cache/cache.go index 04d397711..fee21bc0e 100644 --- a/snapshotter/demux/cache/cache.go +++ b/snapshotter/demux/cache/cache.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/snapshots" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" "github.com/hashicorp/go-multierror" @@ -26,7 +27,7 @@ import ( ) // SnapshotterProvider defines a snapshotter fetch function. -type SnapshotterProvider = func(context.Context, string) (*proxy.RemoteSnapshotter, error) +type SnapshotterProvider = func(context.Context, proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) // RemoteSnapshotterCache implements a cache for remote snapshotters. type RemoteSnapshotterCache struct { @@ -101,8 +102,8 @@ func (c *RemoteSnapshotterCache) List() []string { return keys } -// Get fetches and caches the snapshotter for a given key. -func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +// Put creates and adds a snapshotter to the cache. +func (c *RemoteSnapshotterCache) Put(ctx context.Context, key string, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { c.mutex.RLock() snapshotter, ok := c.snapshotters[key] c.mutex.RUnlock() @@ -113,7 +114,7 @@ func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.Re c.mutex.Unlock() if !ok { - newSnapshotter, err := c.fetch(ctx, key) + newSnapshotter, err := c.fetch(ctx, config) if err != nil { return nil, err } @@ -131,6 +132,18 @@ func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.Re return snapshotter, nil } +// Get retrieves the snapshotter for a given key. +func (c *RemoteSnapshotterCache) Get(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { + c.mutex.RLock() + snapshotter, ok := c.snapshotters[key] + c.mutex.RUnlock() + + if !ok { + return nil, fmt.Errorf("error getting snapshotter for %s: %w", key, errdefs.ErrNotFound) + } + return snapshotter, nil +} + // WalkAll applies the provided function to all cached snapshotters. func (c *RemoteSnapshotterCache) WalkAll(ctx context.Context, fn snapshots.WalkFunc, filters ...string) error { c.mutex.RLock() diff --git a/snapshotter/demux/cache/cache_test.go b/snapshotter/demux/cache/cache_test.go index abf10b010..b9f72cebe 100644 --- a/snapshotter/demux/cache/cache_test.go +++ b/snapshotter/demux/cache/cache_test.go @@ -19,6 +19,7 @@ import ( "fmt" "testing" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/snapshots" "github.com/hashicorp/go-multierror" @@ -26,22 +27,22 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" ) -func getSnapshotterOkFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotterOkFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.SuccessfulSnapshotter{}}, nil } -func getSnapshotterErrorFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotterErrorFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return nil, errors.New("Mock retrieve snapshotter error") } -func getFailingSnapshotterOkFunction(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getFailingSnapshotterOkFunction(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.FailingSnapshotter{}}, nil } func getSnapshotterFromEmptyCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) _, err := uut.Get(context.Background(), "SnapshotterKey") - if err != nil { + if err != nil && !errors.Is(err, errdefs.ErrNotFound) { return fmt.Errorf("Fetch from empty cache incorrectly resulted in error: %w", err) } return nil @@ -49,7 +50,7 @@ func getSnapshotterFromEmptyCache() error { func getCachedSnapshotter() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -81,10 +82,10 @@ func applyWalkFunctionOnEmptyCache() error { func applyWalkFunctionToAllCachedSnapshotters() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "Snapshotter-A"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-A", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter A to empty cache incorrectly resulted in error: %w", err) } - if _, err := uut.Get(context.Background(), "Snapshotter-B"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-B", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter B to cache incorrectly resulted in error: %w", err) } if err := uut.WalkAll(context.Background(), successfulWalk); err != nil { @@ -95,7 +96,7 @@ func applyWalkFunctionToAllCachedSnapshotters() error { func applyWalkFunctionPropagatesErrors() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "Snapshotter-A"); err != nil { + if _, err := uut.Put(context.Background(), "Snapshotter-A", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter A to empty cache incorrectly resulted in error: %w", err) } // The failing snapshotter mock will fail all Walk calls before applying @@ -120,7 +121,7 @@ func evictSnapshotterFromEmptyCache() error { func evictSnapshotterFromCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -132,7 +133,7 @@ func evictSnapshotterFromCache() error { func evictSnapshotterFromCachePropagatesCloseError() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - if _, err := uut.Get(context.Background(), "SnapshotterKey"); err != nil { + if _, err := uut.Put(context.Background(), "SnapshotterKey", proxy.RemoteSnapshotterConfig{}); err != nil { return fmt.Errorf("Adding snapshotter to empty cache incorrectly resulted in error: %w", err) } @@ -162,8 +163,8 @@ func closeCacheWithNonEmptyCache() error { func closeCacheReturnsAllOccurringErrors() error { uut := NewRemoteSnapshotterCache(getFailingSnapshotterOkFunction) - uut.Get(context.Background(), "FailingSnapshotterKey") - uut.Get(context.Background(), "AnotherFailingSnapshotterKey") + uut.Put(context.Background(), "FailingSnapshotterKey", proxy.RemoteSnapshotterConfig{}) + uut.Put(context.Background(), "AnotherFailingSnapshotterKey", proxy.RemoteSnapshotterConfig{}) err := uut.Close() if err == nil { @@ -189,7 +190,7 @@ func listEmptyCache() error { func listNonEmptyCache() error { uut := NewRemoteSnapshotterCache(getSnapshotterOkFunction) - uut.Get(context.Background(), "OkSnapshotterKey") + uut.Put(context.Background(), "OkSnapshotterKey", proxy.RemoteSnapshotterConfig{}) keys := uut.List() if len(keys) != 1 { diff --git a/snapshotter/demux/cache/evict_test.go b/snapshotter/demux/cache/evict_test.go index 8145991ab..2b7061ab7 100644 --- a/snapshotter/demux/cache/evict_test.go +++ b/snapshotter/demux/cache/evict_test.go @@ -26,11 +26,11 @@ import ( "go.uber.org/goleak" ) -func getSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.SuccessfulSnapshotter{}}, nil } -func getErrorSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func getErrorSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return &proxy.RemoteSnapshotter{Snapshotter: &internal.FailingSnapshotter{}}, nil } @@ -52,7 +52,7 @@ func TestCacheEvictionOnConnectionFailure(t *testing.T) { cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) @@ -88,7 +88,7 @@ func TestCacheNotEvictedIfConnectionIsHealthy(t *testing.T) { cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) @@ -121,7 +121,7 @@ func TestBackgroundEnforcersCanBeStopped(t *testing.T) { dialer := proxy.Dialer{Dial: dial, Timeout: 1 * time.Second} cache := NewRemoteSnapshotterCache(getSnapshotter, EvictOnConnectionFailure(dialer, frequency)) - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") cache.Close() @@ -140,7 +140,7 @@ func TestLogErrorOnEvictionFailure(t *testing.T) { cache := NewRemoteSnapshotterCache(getErrorSnapshotter, EvictOnConnectionFailure(dialer, frequency)) defer cache.Close() - _, err := cache.Get(context.Background(), "test") + _, err := cache.Put(context.Background(), "test", proxy.RemoteSnapshotterConfig{}) require.NoError(t, err, "Snapshotter not added to cache correctly") ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) diff --git a/snapshotter/demux/opt.go b/snapshotter/demux/opt.go new file mode 100644 index 000000000..3b9ade525 --- /dev/null +++ b/snapshotter/demux/opt.go @@ -0,0 +1,49 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package demux + +import ( + "encoding/json" + "strconv" + + "github.com/containerd/containerd/snapshots" +) + +// WithVSockPath sets the Firecracker vsock path +func WithVSockPath(vsockPath string) snapshots.Opt { + return snapshots.WithLabels(map[string]string{VSockPathLabel: vsockPath}) +} + +// WithRemoteSnapshotterPort sets the vsock port that the remote-snapshotter +// is listening on for its snapshot server +func WithRemoteSnapshotterPort(port uint32) snapshots.Opt { + return snapshots.WithLabels(map[string]string{RemoteSnapshotterPortLabel: strconv.FormatUint(uint64(port), 10)}) +} + +// WithProxyMetricsPort sets the vsock port that the remote-snapshotter +// is listening on for its metrics server +func WithProxyMetricsPort(port uint32) snapshots.Opt { + return snapshots.WithLabels(map[string]string{MetricsPortLabel: strconv.FormatUint(uint64(port), 10)}) +} + +// WithProxyMetricLabels sets labels that will be attached to proxy metrics +func WithProxyMetricLabels(labels map[string]string) (snapshots.Opt, error) { + serialized, err := json.Marshal(labels) + if err != nil { + return nil, err + } + return snapshots.WithLabels(map[string]string{ + MetricsLabelsLabel: string(serialized), + }), nil +} diff --git a/snapshotter/demux/proxy/address/http_resolver.go b/snapshotter/demux/proxy/address/http_resolver.go deleted file mode 100644 index 19f009338..000000000 --- a/snapshotter/demux/proxy/address/http_resolver.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" -) - -// HTTPClient defines the interface for the client used -// for HTTP communications in the resolver. -type HTTPClient interface { - Get(string) (*http.Response, error) -} - -// ResponseReader defines the reading function interface. -type ResponseReader func(io.Reader) ([]byte, error) - -// HTTPResolver implements a proxy address resolver via HTTP. -type HTTPResolver struct { - url string - client HTTPClient -} - -// NewHTTPResolver creates a new instance of HttpResolver with specified the URL. -func NewHTTPResolver(url string) HTTPResolver { - return HTTPResolver{url: url, client: http.DefaultClient} -} - -func requestURL(url string, namespace string) string { - return fmt.Sprintf("%s/address?namespace=%s", url, namespace) -} - -// Get queries the proxy network type and address for the specified namespace. -func (h HTTPResolver) Get(namespace string) (Response, error) { - url := requestURL(h.url, namespace) - - httpResponse, err := h.client.Get(url) - if err != nil { - return Response{}, err - } - defer httpResponse.Body.Close() - - body, err := ioutil.ReadAll(httpResponse.Body) - if err != nil { - return Response{}, err - } - - code := httpResponse.StatusCode - if code != 200 { - return Response{}, fmt.Errorf("failed to GET %s: status=%d, body=%s", url, code, body) - } - - var response Response - err = json.Unmarshal(body, &response) - if err != nil { - return Response{}, err - } - return response, nil -} diff --git a/snapshotter/demux/proxy/address/http_resolver_test.go b/snapshotter/demux/proxy/address/http_resolver_test.go deleted file mode 100644 index a8b1676c8..000000000 --- a/snapshotter/demux/proxy/address/http_resolver_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -import ( - "errors" - "fmt" - "io" - "net/http" - "testing" -) - -func returnErrorOnHTTPClientError() error { - client := mockClient{} - client.getError = errors.New("error on HTTP GET") - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on HTTP client GET") - } - - return nil -} - -func returnErrorOnResponseReadError() error { - reader := mockReader{response: nil, err: errors.New("mock error on read")} - client := mockClient{getError: nil, getResponse: http.Response{Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on body read") - } - - return nil -} - -func returnErrorOnJSONParserError() error { - reader := mockReader{response: []byte(`{"network": "unix"`), err: io.EOF} - client := mockClient{getError: nil, getResponse: http.Response{Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - if _, err := uut.Get("namespace"); err == nil { - return errors.New("expected error on JSON parsing") - } - - return nil -} - -func happyPath() error { - reader := mockReader{response: []byte( - `{ - "network": "unix", - "address": "/path/to/snapshotter.vsock", - "snapshotter_port": "10000", - "metrics_port": "10001", - "labels": { - "test1": "label1", - "test2": "label2" - } - }`), err: io.EOF} - client := mockClient{getError: nil, getResponse: http.Response{StatusCode: 200, Body: &reader}} - uut := HTTPResolver{url: "localhost:10001", client: &client} - - actual, err := uut.Get("namespace") - if err != nil { - return fmt.Errorf("expected no error from HTTP resolver, but got %+v", err) - } - - if actual.Network != "unix" { - return fmt.Errorf("Expected network 'unix' but actual %s", actual.Address) - } - if actual.Address != "/path/to/snapshotter.vsock" { - return fmt.Errorf("Expected address '/path/to/snapshotter.vsock' but actual %s", actual.Address) - } - if actual.SnapshotterPort != "10000" { - return fmt.Errorf("Expected metrics port '10000' but actual %s", actual.MetricsPort) - } - if actual.MetricsPort != "10001" { - return fmt.Errorf("Expected metrics port '10001' but actual %s", actual.MetricsPort) - } - if actual.Labels["test1"] != "label1" { - return fmt.Errorf("Expected metrics label key='test1' value='label1' but actual %s", actual.Labels["test1"]) - } - if actual.Labels["test2"] != "label2" { - return fmt.Errorf("Expected metrics label key='test2' value='label2' but actual %s", actual.Labels["test1"]) - } - return nil -} - -func TestHttpResolverGet(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - run func() error - }{ - {"HttpClientError", returnErrorOnHTTPClientError}, - {"ResponseReaderError", returnErrorOnResponseReadError}, - {"JsonParserError", returnErrorOnJSONParserError}, - {"HappyPath", happyPath}, - } - - for _, test := range tests { - if err := test.run(); err != nil { - t.Fatalf(test.name+" test failed: %s", err) - } - } -} - -func TestRequestUrlFormat(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - url string - namespace string - expected string - }{ - {"NS-1", "http://127.0.0.1:10001", "ns-1", "http://127.0.0.1:10001/address?namespace=ns-1"}, - {"NS-2", "http://localhost:10001", "ns-2", "http://localhost:10001/address?namespace=ns-2"}, - } - - for _, test := range tests { - if actual := requestURL(test.url, test.namespace); actual != test.expected { - t.Fatalf("%s failed: expected %s actual %s", test.name, test.expected, actual) - } - } -} - -type mockClient struct { - getResponse http.Response - getError error -} - -func (c *mockClient) Get(url string) (*http.Response, error) { - if c.getError != nil { - return nil, c.getError - } - return &c.getResponse, nil -} - -type mockReader struct { - response []byte - err error -} - -func (r *mockReader) Read(p []byte) (int, error) { - return copy(p, r.response), r.err -} - -func (r *mockReader) Close() error { - return nil -} diff --git a/snapshotter/demux/proxy/address/resolver.go b/snapshotter/demux/proxy/address/resolver.go deleted file mode 100644 index e3025944f..000000000 --- a/snapshotter/demux/proxy/address/resolver.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package address - -// Response to the proxy network address resolver. -type Response struct { - // Network type used in net.Dial. - // - // Reference: https://pkg.go.dev/net#Dial - Network string `json:"network"` - - // Network address used in net.Dial. - Address string `json:"address"` - - // SnapshotterPort is the port used in vsock.DialContext for sending snapshotter API requests to the remote snapshotter. - SnapshotterPort string `json:"snapshotter_port"` - - // MetricsPort is the port used in vsock.DialContext for sending metrics requests to the remote snapshotter. - MetricsPort string `json:"metrics_port"` - - // Labels is a map used for applying labels to metrics. - Labels map[string]string `json:"labels"` -} - -// Resolver for the proxy network address. -type Resolver interface { - // Fetch the network address used to forward snapshotter - // requests by namespace lookup from a remote service call. - Get(namespace string) (Response, error) -} diff --git a/snapshotter/demux/proxy/snapshotter.go b/snapshotter/demux/proxy/snapshotter.go index 39315128f..08906fc23 100644 --- a/snapshotter/demux/proxy/snapshotter.go +++ b/snapshotter/demux/proxy/snapshotter.go @@ -36,6 +36,14 @@ type Dialer struct { Timeout time.Duration } +// RemoteSnapshotterConfig is the configuration needed to create a new remote snapshotter. +type RemoteSnapshotterConfig struct { + VSockPath string + RemoteSnapshotterPort uint32 + MetricsPort uint32 + MetricsLabels map[string]string +} + // RemoteSnapshotter embeds a snapshots.Snapshotter and its metrics proxy. type RemoteSnapshotter struct { snapshots.Snapshotter diff --git a/snapshotter/demux/snapshotter.go b/snapshotter/demux/snapshotter.go index 96c33902a..bb126a150 100644 --- a/snapshotter/demux/snapshotter.go +++ b/snapshotter/demux/snapshotter.go @@ -15,7 +15,12 @@ package demux import ( "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" @@ -25,9 +30,23 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/vm" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/cache" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" + mountutil "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/mount" ) +const ( + // VSockPathLabel is a snapshot label that contains the VM's vsock path. + VSockPathLabel = "containerd.io/snapshot/remote/demux.vsock.address" + // RemoteSnapshotterPortLabel is a snapshot label that contains the vsock port the remote snapshotter is listening on inside the VM. + RemoteSnapshotterPortLabel = "containerd.io/snapshot/remote/demux.vsock.port" + // MetricsPortLabel is a snapshot label that contains the vsock port the remote snapshotter's metrics server is listening on inside the VM. + MetricsPortLabel = "containerd.io/snapshot/remote/demux.metrics.port" + // MetricsLabelsLabel is a snapshot label that contains the set of labels to attach to metrics when scraping. + MetricsLabelsLabel = "containerd.io/snapshot/remote/demux.metrics.labels" +) + +var errNoVsockPath = fmt.Errorf("the snapshot is missing the %s label", VSockPathLabel) + // Snapshotter routes snapshotter requests to their destined // remote snapshotter via their snapshotter namespace. // @@ -109,7 +128,17 @@ func (s *Snapshotter) Prepare(ctx context.Context, key string, parent string, op snapshotter, err := s.getSnapshotterFromCache(ctx, contextLogger) if err != nil { - return []mount.Mount{}, err + if !errors.Is(err, errdefs.ErrNotFound) { + return nil, err + } + snapshotterConfig, err := extractRemoteSnapshotterConfig(opts...) + if err != nil { + return nil, err + } + snapshotter, err = s.putSnapshotterIntoCache(ctx, snapshotterConfig, contextLogger) + if err != nil { + return nil, err + } } mounts, err := snapshotter.Prepare(ctx, key, parent, opts...) @@ -221,6 +250,19 @@ func (s *Snapshotter) getSnapshotterFromCache(ctx context.Context, log *logrus.E return snapshotter, nil } +func (s *Snapshotter) putSnapshotterIntoCache(ctx context.Context, snapshotterConfig proxy.RemoteSnapshotterConfig, log *logrus.Entry) (*proxy.RemoteSnapshotter, error) { + namespace, err := getNamespaceFromContext(ctx, log) + if err != nil { + return nil, err + } + snapshotter, err := s.cache.Put(ctx, namespace, snapshotterConfig) + if err != nil { + log.WithField("namespace", namespace).WithError(err).Error(snapshotterNotFoundErrorString) + return nil, err + } + return snapshotter, nil +} + const missingNamespaceErrorString = "Function called without namespaced context" func getNamespaceFromContext(ctx context.Context, log *logrus.Entry) (string, error) { @@ -231,3 +273,56 @@ func getNamespaceFromContext(ctx context.Context, log *logrus.Entry) (string, er } return namespace, nil } + +func extractRemoteSnapshotterConfig(opts ...snapshots.Opt) (proxy.RemoteSnapshotterConfig, error) { + var base snapshots.Info + for _, opt := range opts { + if err := opt(&base); err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + // extract vsock path + path, ok := base.Labels[VSockPathLabel] + if !ok { + return proxy.RemoteSnapshotterConfig{}, errNoVsockPath + } + + // extract remote snapshotter port + snapshotterPortStr, ok := base.Labels[RemoteSnapshotterPortLabel] + if !ok { + return proxy.RemoteSnapshotterConfig{}, fmt.Errorf("the snapshot is missing the %s label", RemoteSnapshotterPortLabel) + } + snapshotterPort, err := strconv.Atoi(snapshotterPortStr) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + + // extract metrics port + metricsPortStr, ok := base.Labels[MetricsPortLabel] + var metricsPort int + if ok { + metricsPort, err = strconv.Atoi(metricsPortStr) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + // extract metrics labels + var metricsLabels map[string]string + metricsLabelsSerialied := base.Labels[MetricsLabelsLabel] + if metricsLabelsSerialied != "" { + err := json.Unmarshal([]byte(metricsLabelsSerialied), &metricsLabels) + if err != nil { + return proxy.RemoteSnapshotterConfig{}, err + } + } + + return proxy.RemoteSnapshotterConfig{ + VSockPath: path, + RemoteSnapshotterPort: uint32(snapshotterPort), + MetricsPort: uint32(metricsPort), + MetricsLabels: metricsLabels, + }, nil + +} diff --git a/snapshotter/demux/snapshotter_test.go b/snapshotter/demux/snapshotter_test.go index 06a1660f7..3fa76276a 100644 --- a/snapshotter/demux/snapshotter_test.go +++ b/snapshotter/demux/snapshotter_test.go @@ -27,29 +27,18 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" ) -func fetchOkSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func fetchOkSnapshotter(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { var snapshotter internal.SuccessfulSnapshotter = internal.SuccessfulSnapshotter{} return &proxy.RemoteSnapshotter{Snapshotter: &snapshotter}, nil } -func fetchSnapshotterNotFound(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { +func fetchSnapshotterNotFound(ctx context.Context, config proxy.RemoteSnapshotterConfig) (*proxy.RemoteSnapshotter, error) { return nil, errors.New("mock snapshotter not found") } func createSnapshotterCacheWithSuccessfulSnapshotter(namespace string) *cache.RemoteSnapshotterCache { cache := cache.NewRemoteSnapshotterCache(fetchOkSnapshotter) - cache.Get(context.Background(), namespace) - return cache -} - -func fetchFailingSnapshotter(ctx context.Context, key string) (*proxy.RemoteSnapshotter, error) { - var snapshotter internal.FailingSnapshotter = internal.FailingSnapshotter{} - return &proxy.RemoteSnapshotter{Snapshotter: &snapshotter}, nil -} - -func createSnapshotterCacheWithFailingSnapshotter(namespace string) *cache.RemoteSnapshotterCache { - cache := cache.NewRemoteSnapshotterCache(fetchFailingSnapshotter) - cache.Get(context.Background(), namespace) + cache.Put(context.Background(), namespace, proxy.RemoteSnapshotterConfig{}) return cache } @@ -144,45 +133,6 @@ func TestReturnErrorWhenSnapshotterNotFound(t *testing.T) { } } -func TestReturnErrorAfterProxyFunctionFailure(t *testing.T) { - t.Parallel() - - const namespace = "testing" - cache := createSnapshotterCacheWithFailingSnapshotter(namespace) - ctx := namespaces.WithNamespace(context.Background(), namespace) - ctx = logtest.WithT(ctx, t) - - uut := NewSnapshotter(cache) - - tests := []struct { - name string - run func() error - }{ - {"Stat", func() error { _, err := uut.Stat(ctx, "layerKey"); return err }}, - {"Update", func() error { _, err := uut.Update(ctx, snapshots.Info{}); return err }}, - {"Usage", func() error { _, err := uut.Usage(ctx, "layerKey"); return err }}, - {"Mounts", func() error { _, err := uut.Mounts(ctx, "layerKey"); return err }}, - {"Prepare", func() error { _, err := uut.Prepare(ctx, "layerKey", ""); return err }}, - {"View", func() error { _, err := uut.View(ctx, "layerKey", ""); return err }}, - {"Commit", func() error { return uut.Commit(ctx, "layer1", "layerKey") }}, - {"Remove", func() error { return uut.Remove(ctx, "layerKey") }}, - {"Walk", func() error { - var callback = func(c context.Context, i snapshots.Info) error { return nil } - return uut.Walk(ctx, callback) - }}, - {"Cleanup", func() error { return uut.(snapshots.Cleaner).Cleanup(ctx) }}, - {"Close", func() error { return uut.Close() }}, - } - - for _, test := range tests { - t.Run(test.name+"ProxyFailure", func(t *testing.T) { - if err := test.run(); err == nil { - t.Fatalf("%s call did not return error", test.name) - } - }) - } -} - func TestNoErrorIsReturnedOnSuccessfulProxyExecution(t *testing.T) { t.Parallel() diff --git a/snapshotter/internal/http_address_resolver.go b/snapshotter/internal/http_address_resolver.go deleted file mode 100644 index 5c5e64a9a..000000000 --- a/snapshotter/internal/http_address_resolver.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - - "github.com/containerd/containerd/namespaces" - "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" - - "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" - "github.com/firecracker-microvm/firecracker-containerd/proto" - proxyaddress "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy/address" -) - -var ( - port int - remotePort int - metricsRemotePort int - containerdSockPath string - logger *logrus.Logger -) - -func init() { - flag.IntVar(&port, "port", 10001, "service port for address resolver") - flag.StringVar(&containerdSockPath, "containerdSocket", "/run/firecracker-containerd/containerd.sock", "filepath to the containerd socket") - flag.IntVar(&remotePort, "remotePort", 10000, "the remote port on which the remote snapshotter is listening") - flag.IntVar(&metricsRemotePort, "metricsRemotePort", 10002, "the remote port on which the remote snapshotter metrics server is listening") - logger = logrus.New() -} - -// Simple example of an HTTP service to resolve snapshotter namespace -// to a forwarding address for the demultiplexing snapshotter. -// -// Example: -// curl -X GET "http://localhost:10001/address?namespace=ns-1" -// -// Response: -// { -// "network": "unix", -// "address": "/var/lib/firecracker-containerd/shim-base/default#cbfad871-0862-4dd6-ae7a-52e9b1c16ede/firecracker.vsock" -// } -func main() { - ctx, cancel := context.WithCancel(context.Background()) - - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - - <-c - cancel() - }() - - if !flag.Parsed() { - flag.Parse() - } - - http.HandleFunc("/address", queryAddress) - httpServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - } - - logger.Info(fmt.Sprintf("http resolver serving at port %d", port)) - g, gCtx := errgroup.WithContext(ctx) - g.Go(func() error { - return httpServer.ListenAndServe() - }) - g.Go(func() error { - <-gCtx.Done() - return httpServer.Shutdown(context.Background()) - }) - - if err := g.Wait(); err != http.ErrServerClosed { - logger.WithError(err).Error() - os.Exit(1) - } - - logger.Info("http: server closed") -} - -func queryAddress(writ http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - http.Error(writ, fmt.Sprintf("%s method not allowed", req.Method), http.StatusForbidden) - return - } - - writ.Header().Set("Content-Type", "application/json") - - keys, ok := req.URL.Query()["namespace"] - if !ok { - http.Error(writ, "Missing 'namespace' query", http.StatusBadRequest) - return - } - - sock := containerdSockPath + ".ttrpc" - fcClient, err := client.New(sock) - if err != nil { - logger.WithError(err).Error("could not create firecracker client") - http.Error(writ, fmt.Sprintf("failed to connect %q", sock), http.StatusInternalServerError) - return - } - defer fcClient.Close() - - namespace := keys[0] - ctx := namespaces.WithNamespace(req.Context(), namespace) - vmInfo, err := fcClient.GetVMInfo(ctx, &proto.GetVMInfoRequest{VMID: namespace}) - if err != nil { - logger.WithField("VMID", namespace).WithError(err).Error("unable to retrieve VM Info") - http.Error(writ, fmt.Sprintf("failed to get VM %q", namespace), http.StatusNotFound) - return - } - - writ.WriteHeader(http.StatusOK) - - result := proxyaddress.Response{ - Network: "unix", - Address: vmInfo.VSockPath, - SnapshotterPort: strconv.Itoa(remotePort), - MetricsPort: strconv.Itoa(metricsRemotePort), - Labels: map[string]string{ - "VMID": namespace, - }, - } - serialized, err := json.Marshal(&result) - if err != nil { - http.Error(writ, fmt.Sprintf("failed to marshal %+v", result), http.StatusInternalServerError) - return - } - writ.Write(serialized) -} diff --git a/snapshotter/metrics_integ_test.go b/snapshotter/metrics_integ_test.go index 63bfe0ad9..b42e05ed9 100644 --- a/snapshotter/metrics_integ_test.go +++ b/snapshotter/metrics_integ_test.go @@ -27,6 +27,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -91,7 +92,7 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie if err != nil { return fmt.Errorf("Unable to create client to containerd service at %s, is containerd running?: %v", integtest.ContainerdSockPath, err) } - if _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, RootDrive: &proto.FirecrackerRootDrive{ HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img", @@ -110,7 +111,8 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie MemSizeMib: 512, }, ContainerCount: 1, - }); err != nil { + }) + if err != nil { return fmt.Errorf("Failed to create microVM[%s]: %v", vmID, err) } @@ -123,7 +125,11 @@ func pullImageWithRemoteSnapshotterInVM(ctx context.Context, vmID string, fcClie image, err := client.Pull(ctx, al2stargz, containerd.WithPullUnpack, - containerd.WithPullSnapshotter(snapshotterName), + containerd.WithPullSnapshotter(snapshotterName, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + demux.WithProxyMetricsPort(10002), + ), containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*1024*1024)), ) if err != nil { @@ -162,7 +168,7 @@ func verifyMetricsResponse(t *testing.T, numberOfVms int) { uniqueTargets[mt.Targets[0]] = struct{}{} metricsResponse, err := http.Get("http://" + mt.Targets[0] + "/metrics") require.NoError(t, err, "Failed to get metrics proxy") - require.Equal(t, metricsResponse.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, metricsResponse.StatusCode) defer metricsResponse.Body.Close() mBytes, err := io.ReadAll(metricsResponse.Body) diff --git a/snapshotter/service_integ_test.go b/snapshotter/service_integ_test.go index 7882352bb..0841407d4 100644 --- a/snapshotter/service_integ_test.go +++ b/snapshotter/service_integ_test.go @@ -25,6 +25,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -93,7 +94,7 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string) // https://github.com/firecracker-microvm/firecracker/blob/v1.1.0/docs/prod-host-setup.md kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs + " 8250.nr_uarts=0 quiet loglevel=1" - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, KernelArgs: kernelArgs, RootDrive: &proto.FirecrackerRootDrive{ @@ -130,7 +131,10 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string) image, err := client.Pull(ctx, al2stargz, containerd.WithPullUnpack, - containerd.WithPullSnapshotter(snapshotterName), + containerd.WithPullSnapshotter(snapshotterName, + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib)), ) if err != nil { diff --git a/snapshotter/volume_integ_test.go b/snapshotter/volume_integ_test.go index 006a139aa..1e895b2fd 100644 --- a/snapshotter/volume_integ_test.go +++ b/snapshotter/volume_integ_test.go @@ -24,6 +24,7 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" "github.com/firecracker-microvm/firecracker-containerd/proto" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" + "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux" "github.com/firecracker-microvm/firecracker-containerd/snapshotter/internal/integtest/stargz/fs/source" "github.com/firecracker-microvm/firecracker-containerd/volume" "github.com/stretchr/testify/assert" @@ -56,24 +57,12 @@ func TestGuestVolumeFrom_Isolated(t *testing.T) { err = vs.AddFrom(ctx, localImage) require.NoError(t, err) - // Add a stargz image. - // The volume directories must be specified since the host's containerd doesn't know about the image. - remoteImage := volume.FromGuestImage( - client, vmID, al2stargz, "al2-snapshot", []string{"/etc/yum"}, - volume.WithSnapshotter("demux"), - volume.WithPullOptions(containerd.WithImageHandlerWrapper( - source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib), - )), - ) - err = vs.AddFrom(ctx, remoteImage) - require.NoError(t, err) - // PrepareDriveMount only copies images that are available before starting the VM. // In this case, only postgres. mount, err := vs.PrepareDriveMount(ctx, 10*mib) require.NoError(t, err) - _, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{ + vminfo, err := fcClient.CreateVM(ctx, &proto.CreateVMRequest{ VMID: vmID, RootDrive: &proto.FirecrackerRootDrive{ HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img", @@ -97,6 +86,22 @@ func TestGuestVolumeFrom_Isolated(t *testing.T) { require.NoErrorf(t, err, "Failed to create microVM[%s]", vmID) defer fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: vmID}) + // Add a stargz image. + // The volume directories must be specified since the host's containerd doesn't know about the image. + remoteImage := volume.FromGuestImage( + client, vmID, al2stargz, "al2-snapshot", []string{"/etc/yum"}, + volume.WithSnapshotter("demux"), + volume.WithPullOptions(containerd.WithImageHandlerWrapper( + source.AppendDefaultLabelsHandlerWrapper(al2stargz, 10*mib), + )), + volume.WithSnapshotOptions( + demux.WithVSockPath(vminfo.VSockPath), + demux.WithRemoteSnapshotterPort(10000), + ), + ) + err = vs.AddFrom(ctx, remoteImage) + require.NoError(t, err) + _, err = fcClient.SetVMMetadata(ctx, &proto.SetVMMetadataRequest{ VMID: vmID, Metadata: fmt.Sprintf(dockerMetadataTemplate, "ghcr.io", noAuth, noAuth), diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh index 77db62128..71fcabfbe 100755 --- a/tools/docker/entrypoint.sh +++ b/tools/docker/entrypoint.sh @@ -46,10 +46,6 @@ cat > /etc/demux-snapshotter/config.toml <> ${FICD_CONTAINERD_OUTFILE} & -/usr/local/bin/http-address-resolver &>> ${FICD_LOG_DIR}/http-address-resolver.out & /usr/local/bin/demux-snapshotter &>> ${FICD_LOG_DIR}/demux-snapshotter.out & exec /bin/bash -c "$@" diff --git a/volume/.gitignore b/volume/.gitignore new file mode 100644 index 000000000..3084a1dc7 --- /dev/null +++ b/volume/.gitignore @@ -0,0 +1,2 @@ +volume-init + diff --git a/volume/guest_image.go b/volume/guest_image.go index 43a6b42d1..e224c3cc5 100644 --- a/volume/guest_image.go +++ b/volume/guest_image.go @@ -90,7 +90,7 @@ func (p *GuestVolumeImageProvider) Name() string { func (p *GuestVolumeImageProvider) pull(ctx context.Context) error { remoteOpts := []containerd.RemoteOpt{ containerd.WithPullUnpack, - containerd.WithPullSnapshotter(p.config.snapshotter), + containerd.WithPullSnapshotter(p.config.snapshotter, p.config.snapshotOpts...), } remoteOpts = append(remoteOpts, p.config.pullOpts...) image, err := p.client.Pull(ctx, p.image, remoteOpts...) diff --git a/volume/image.go b/volume/image.go index 41cb80b77..7cf2eae68 100644 --- a/volume/image.go +++ b/volume/image.go @@ -32,8 +32,9 @@ import ( ) type imageProviderConfig struct { - snapshotter string - pullOpts []containerd.RemoteOpt + snapshotter string + pullOpts []containerd.RemoteOpt + snapshotOpts []snapshots.Opt } type imageVolumeProvider struct { @@ -61,6 +62,13 @@ func WithPullOptions(opts ...containerd.RemoteOpt) ImageOpt { } } +// WithSnapshotOptions sets the snapshotter's snapshot options. +func WithSnapshotOptions(opts ...snapshots.Opt) ImageOpt { + return func(c *imageProviderConfig) { + c.snapshotOpts = opts + } +} + func getV1ImageConfig(ctx context.Context, image containerd.Image) (*v1.Image, error) { var result v1.Image