搭建过docker registry的人都知道,docker默认不支持http的registry,如果一定要支持,就需要配置--insecure-registry
选项才可以,而且配置完以后需要重启docker daemon。本文从源代码角度分析docker daemon是如何限制的,以及如何去掉这个限制。
我们搭建了一个私有的镜像仓库,地址是http://222.222.222.222,没有配置https。因为要支持http,需要配置docker daemon,所以基本可以判断是docker daemon做的限制。我们通过docker login
登录http://222.222.222.222时docker daemon的日志会报如下错误:
time="2017-01-16T09:30:33.349460061Z" level=debug msg="Calling POST /v1.24/auth"
time="2017-01-16T09:30:33.350468050Z" level=debug msg="form data: {\"password\":\"*****\",\"serveraddress\":\"http://222.222.222.222/\",\"username\":\"test\"}"
time="2017-01-16T09:30:33.351098322Z" level=debug msg="hostDir: /etc/docker/certs.d/222.222.222.222"
time="2017-01-16T09:30:33.351607498Z" level=debug msg="hostDir: /etc/docker/certs.d/222.222.222.222"
time="2017-01-16T09:30:33.353994569Z" level=debug msg="attempting v2 login to registry endpoint https://222.222.222.222/v2/"
time="2017-01-16T09:30:33.383185881Z" level=info msg="Error logging in to v2 endpoint, trying next endpoint: Get https://222.222.222.222/v2/: dial tcp 222.222.222.222:443: getsockopt: connection refused"
time="2017-01-16T09:30:33.383694112Z" level=debug msg="attempting v1 login to registry endpoint https://222.222.222.222/v1/"
time="2017-01-16T09:30:33.411921859Z" level=info msg="Error logging in to v1 endpoint, trying next endpoint: Get https://222.222.222.222/v1/users/: dial tcp 222.222.222.222:443: getsockopt: connection refused"
time="2017-01-16T09:30:33.412569676Z" level=error msg="Handler for POST /v1.24/auth returned error: Get https://222.222.222.222/v1/users/: dial tcp 222.222.222.222:443: getsockopt: connection refused"
然后我们给docker daemon配置了--insecure-registry
选项后登录,观察日志:
time="2017-01-17T09:56:52.267456792Z" level=debug msg="Calling POST /v1.24/auth"
time="2017-01-17T09:56:52.268047734Z" level=debug msg="form data: {\"password\":\"*****\",\"serveraddress\":\"http://222.222.222.222\",\"username\":\"admin\"}"
time="2017-01-17T09:56:52.268603531Z" level=debug msg="attempting v2 login to registry endpoint https://222.222.222.222/v2/"
time="2017-01-17T09:56:52.300374963Z" level=info msg="Error logging in to v2 endpoint, trying next endpoint: Get https://222.222.222.222/v2/: dial tcp 222.222.222.222:443: getsockopt: connection refused"
time="2017-01-17T09:56:52.301470933Z" level=debug msg="attempting v2 login to registry endpoint http://54.222.229.223/v2/"222.222.222.222
我们发现没有配置--insecure-registry
选项前只会尝试https://xxx
,但是配置了以后,还会去尝试http://xxx
。因为我们只有http服务,如果不去尝试http的话,肯定是登录不了的。然后我们去看下docker的代码。
因为是跟认证相关的,所以我们很容易就可以锁定docker/daemon/auth.go
这个文件,文件内容很少:
package daemon
import (
"golang.org/x/net/context"
"github.com/docker/docker/api/types"
"github.com/docker/docker/dockerversion"
)
// AuthenticateToRegistry checks the validity of credentials in authConfig
func (daemon *Daemon) AuthenticateToRegistry(ctx context.Context, authConfig *types.AuthConfig) (string, string, error) {
return daemon.RegistryService.Auth(ctx, authConfig, dockerversion.DockerUserAgent(ctx))
}
RegistryService
的类型是registry.Service
,或者根据代码跳转我们可以很容易的找到docker/registry/service.go
这个文件,因为这个文件对于分析比较重要,这里列出全部内容:
package registry
import (
"crypto/tls"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"golang.org/x/net/context"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/docker/api/types"
registrytypes "github.com/docker/docker/api/types/registry"
"github.com/docker/docker/reference"
)
const (
// DefaultSearchLimit is the default value for maximum number of returned search results.
DefaultSearchLimit = 25
)
// Service is the interface defining what a registry service should implement.
type Service interface {
Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error)
LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error)
LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error)
ResolveRepository(name reference.Named) (*RepositoryInfo, error)
Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error)
ServiceConfig() *registrytypes.ServiceConfig
TLSConfig(hostname string) (*tls.Config, error)
LoadMirrors([]string) error
LoadInsecureRegistries([]string) error
}
// DefaultService is a registry service. It tracks configuration data such as a list
// of mirrors.
type DefaultService struct {
config *serviceConfig
mu sync.Mutex
}
// NewService returns a new instance of DefaultService ready to be
// installed into an engine.
func NewService(options ServiceOptions) *DefaultService {
return &DefaultService{
config: newServiceConfig(options),
}
}
// ServiceConfig returns the public registry service configuration.
func (s *DefaultService) ServiceConfig() *registrytypes.ServiceConfig {
s.mu.Lock()
defer s.mu.Unlock()
servConfig := registrytypes.ServiceConfig{
InsecureRegistryCIDRs: make([]*(registrytypes.NetIPNet), 0),
IndexConfigs: make(map[string]*(registrytypes.IndexInfo)),
Mirrors: make([]string, 0),
}
// construct a new ServiceConfig which will not retrieve s.Config directly,
// and look up items in s.config with mu locked
servConfig.InsecureRegistryCIDRs = append(servConfig.InsecureRegistryCIDRs, s.config.ServiceConfig.InsecureRegistryCIDRs...)
for key, value := range s.config.ServiceConfig.IndexConfigs {
servConfig.IndexConfigs[key] = value
}
servConfig.Mirrors = append(servConfig.Mirrors, s.config.ServiceConfig.Mirrors...)
return &servConfig
}
// LoadMirrors loads registry mirrors for Service
func (s *DefaultService) LoadMirrors(mirrors []string) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.config.LoadMirrors(mirrors)
}
// LoadInsecureRegistries loads insecure registries for Service
func (s *DefaultService) LoadInsecureRegistries(registries []string) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.config.LoadInsecureRegistries(registries)
}
// Auth contacts the public registry with the provided credentials,
// and returns OK if authentication was successful.
// It can be used to verify the validity of a client's credentials.
func (s *DefaultService) Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) {
// TODO Use ctx when searching for repositories
serverAddress := authConfig.ServerAddress
if serverAddress == "" {
serverAddress = IndexServer
}
if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") {
serverAddress = "https://" + serverAddress
}
u, err := url.Parse(serverAddress)
if err != nil {
return "", "", fmt.Errorf("unable to parse server address: %v", err)
}
endpoints, err := s.LookupPushEndpoints(u.Host)
if err != nil {
return "", "", err
}
for _, endpoint := range endpoints {
login := loginV2
if endpoint.Version == APIVersion1 {
login = loginV1
}
status, token, err = login(authConfig, endpoint, userAgent)
if err == nil {
return
}
if fErr, ok := err.(fallbackError); ok {
err = fErr.err
logrus.Infof("Error logging in to %s endpoint, trying next endpoint: %v", endpoint.Version, err)
continue
}
return "", "", err
}
return "", "", err
}
// splitReposSearchTerm breaks a search term into an index name and remote name
func splitReposSearchTerm(reposName string) (string, string) {
nameParts := strings.SplitN(reposName, "/", 2)
var indexName, remoteName string
if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") &&
!strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") {
// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
// 'docker.io'
indexName = IndexName
remoteName = reposName
} else {
indexName = nameParts[0]
remoteName = nameParts[1]
}
return indexName, remoteName
}
// Search queries the public registry for images matching the specified
// search terms, and returns the results.
func (s *DefaultService) Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) {
// TODO Use ctx when searching for repositories
if err := validateNoScheme(term); err != nil {
return nil, err
}
indexName, remoteName := splitReposSearchTerm(term)
// Search is a long-running operation, just lock s.config to avoid block others.
s.mu.Lock()
index, err := newIndexInfo(s.config, indexName)
s.mu.Unlock()
if err != nil {
return nil, err
}
// *TODO: Search multiple indexes.
endpoint, err := NewV1Endpoint(index, userAgent, http.Header(headers))
if err != nil {
return nil, err
}
var client *http.Client
if authConfig != nil && authConfig.IdentityToken != "" && authConfig.Username != "" {
creds := NewStaticCredentialStore(authConfig)
scopes := []auth.Scope{
auth.RegistryScope{
Name: "catalog",
Actions: []string{"search"},
},
}
modifiers := DockerHeaders(userAgent, nil)
v2Client, foundV2, err := v2AuthHTTPClient(endpoint.URL, endpoint.client.Transport, modifiers, creds, scopes)
if err != nil {
if fErr, ok := err.(fallbackError); ok {
logrus.Errorf("Cannot use identity token for search, v2 auth not supported: %v", fErr.err)
} else {
return nil, err
}
} else if foundV2 {
// Copy non transport http client features
v2Client.Timeout = endpoint.client.Timeout
v2Client.CheckRedirect = endpoint.client.CheckRedirect
v2Client.Jar = endpoint.client.Jar
logrus.Debugf("using v2 client for search to %s", endpoint.URL)
client = v2Client
}
}
if client == nil {
client = endpoint.client
if err := authorizeClient(client, authConfig, endpoint); err != nil {
return nil, err
}
}
r := newSession(client, authConfig, endpoint)
if index.Official {
localName := remoteName
if strings.HasPrefix(localName, "library/") {
// If pull "library/foo", it's stored locally under "foo"
localName = strings.SplitN(localName, "/", 2)[1]
}
return r.SearchRepositories(localName, limit)
}
return r.SearchRepositories(remoteName, limit)
}
// ResolveRepository splits a repository name into its components
// and configuration of the associated registry.
func (s *DefaultService) ResolveRepository(name reference.Named) (*RepositoryInfo, error) {
s.mu.Lock()
defer s.mu.Unlock()
return newRepositoryInfo(s.config, name)
}
// APIEndpoint represents a remote API endpoint
type APIEndpoint struct {
Mirror bool
URL *url.URL
Version APIVersion
Official bool
TrimHostname bool
TLSConfig *tls.Config
}
// ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint
func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*V1Endpoint, error) {
return newV1Endpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders)
}
// TLSConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) TLSConfig(hostname string) (*tls.Config, error) {
s.mu.Lock()
defer s.mu.Unlock()
return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}
// tlsConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) tlsConfig(hostname string) (*tls.Config, error) {
return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}
func (s *DefaultService) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) {
return s.tlsConfig(mirrorURL.Host)
}
// LookupPullEndpoints creates a list of endpoints to try to pull from, in order of preference.
// It gives preference to v2 endpoints over v1, mirrors over the actual
// registry, and HTTPS over plain HTTP.
func (s *DefaultService) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.lookupEndpoints(hostname)
}
// LookupPushEndpoints creates a list of endpoints to try to push to, in order of preference.
// It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP.
// Mirrors are not included.
func (s *DefaultService) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
s.mu.Lock()
defer s.mu.Unlock()
allEndpoints, err := s.lookupEndpoints(hostname)
if err == nil {
for _, endpoint := range allEndpoints {
if !endpoint.Mirror {
endpoints = append(endpoints, endpoint)
}
}
}
return endpoints, err
}
func (s *DefaultService) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
endpoints, err = s.lookupV2Endpoints(hostname)
if err != nil {
return nil, err
}
if s.config.V2Only {
return endpoints, nil
}
legacyEndpoints, err := s.lookupV1Endpoints(hostname)
if err != nil {
return nil, err
}
endpoints = append(endpoints, legacyEndpoints...)
return endpoints, nil
}
代码比较简单,里面定义了一个Service
接口,并且定义了一个DefaultService
结构体实现了这个接口。我们主要关注一下Auth
这个函数。细节我就不说了,我说一下和我们要分析的点比较相关的流程。重点在LookupPushEndpoints
这个函数,从代码可以看到是这个函数构造了一个endpoints
,里面是会去尝试登陆的url。那么我们的http那个应该也是在这个里面构造的。
我们进到从LookupPushEndpoints
跟踪到lookupEndpoints
,再到lookupV2Endpoints
。需要说明的是还有一个lookupV1Endpoints
,这个主要是为了兼容以前V1版本的registry,现在已经很少了。而且V1和V2的代码逻辑基本是一样的,我们以V2为例分析。lookupV2Endpoints
在docker/registry/service_v2.go
中:
package registry
import (
"net/url"
"strings"
"github.com/docker/go-connections/tlsconfig"
)
func (s *DefaultService) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) {
tlsConfig := tlsconfig.ServerDefault()
if hostname == DefaultNamespace || hostname == IndexHostname {
// v2 mirrors
for _, mirror := range s.config.Mirrors {
if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") {
mirror = "https://" + mirror
}
mirrorURL, err := url.Parse(mirror)
if err != nil {
return nil, err
}
mirrorTLSConfig, err := s.tlsConfigForMirror(mirrorURL)
if err != nil {
return nil, err
}
endpoints = append(endpoints, APIEndpoint{
URL: mirrorURL,
// guess mirrors are v2
Version: APIVersion2,
Mirror: true,
TrimHostname: true,
TLSConfig: mirrorTLSConfig,
})
}
// v2 registry
endpoints = append(endpoints, APIEndpoint{
URL: DefaultV2Registry,
Version: APIVersion2,
Official: true,
TrimHostname: true,
TLSConfig: tlsConfig,
})
return endpoints, nil
}
tlsConfig, err = s.tlsConfig(hostname)
if err != nil {
return nil, err
}
endpoints = []APIEndpoint{
{
URL: &url.URL{
Scheme: "https",
Host: hostname,
},
Version: APIVersion2,
TrimHostname: true,
TLSConfig: tlsConfig,
},
}
if tlsConfig.InsecureSkipVerify {
endpoints = append(endpoints, APIEndpoint{
URL: &url.URL{
Scheme: "http",
Host: hostname,
},
Version: APIVersion2,
TrimHostname: true,
// used to check if supposed to be secure via InsecureSkipVerify
TLSConfig: tlsConfig,
})
}
return endpoints, nil
}
到这里,我们已经已经看到真想了。可以看到要想生成http的endpoint,需要tlsConfig.InsecureSkipVerify
这个为true才可以。那这个又是在哪里设置的?这段代码里面有两处涉及tlsConfig
的地方:tlsConfig := tlsconfig.ServerDefault()
和tlsConfig, err = s.tlsConfig(hostname)
。但因为那个配置的选项是和hostname相关的,所以肯定是后者。依次进到tlsConfig
、isSecureIndex
,就看到了,在isSecureIndex
里面,如果从配置项里面找到了,就返回该条的Secure
值。这个Secure
字段默认是true,但如果我们配置了--insecure-registry
选项,这个字段就会被置为false,具体实现在LoadInsecureRegistries
函数里面,有兴趣的可以看一下。
我们已经明白了其中的原理,想让docker daemon默认就支持http的registry,改起来就非常简单了,最粗暴的做法就是将tlsConfig.InsecureSkipVerify
这个恒置为true或者直接去掉那里的if,无条件执行下面的代码。当然,我觉得比较优雅的做法是增加一个配置项,比如--accept-insecure-registry
,默认是false,然后再if那里和tlsConfig.InsecureSkipVerify
进行或操作。这样的好处是和原来的相比这个是一种通用的配置,不会增加一项就要配置一次,重启一次docker daemon。另外,如果置为false,那就和docker原来的机制是完全一样的了,灵活性比较大。
2017.1.24更新
再看了下代码发现isSecureIndex
函数里面支持解析CIDR,也就是说我们可以在配置--insecure-registry
时配置子网,那么我们可以将这个选项配置为0.0.0.0/0
就可以支持所有http的registry了。当然,还是那句话:http是不安全的,慎用。就算是私网内,也是可以自签名用https的。
解决了我的问题,谢谢了,还是要细心读啊,终于找到答案。
客气。加班够辛苦的啊