Краткое содержание
У меня есть самостоятельный gitlab, установленный с помощью apt. У меня нет git_data_dirрасположения по умолчанию (см. конфигурацию). Я не могу отправить образ docker в реестр docker, каталог <shared_path>/registryне существует. Никаких миграций, никаких процедур резервного копирования/восстановления, последняя версия.
Проблема, которую нужно решить
GitLab отклоняет Docker image push из удаленного расположения с ошибкой 500. Я не могу push image в свой личный реестр docker. Есть ли у кого-нибудь идеи, почему и как это решить?
Загрузка изображения с удаленного устройства
root@remote:cat Dockerfile
FROM alpine
root@remote:~/playground# docker login gitlab.mydomain.com:5050
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
root@remote:~/playground# docker build -t gitlab.mydomain.com:5050/testing/registry .
Sending build context to Docker daemon 2.048kB
Step 1/1 : FROM alpine
---> e7d92cdc71fe
Successfully built e7d92cdc71fe
Successfully tagged gitlab.mydomain.com:5050/testing/registry:latest
root@remote:~/playground# docker push gitlab.mydomain.com:5050/testing/registry
The push refers to repository [gitlab.mydomain.com:5050/testing/registry]
5216338b40a7: Retrying in 1 second
received unexpected HTTP status: 500 Internal Server Error
Журнал реестра Gitlab
repo:/# tail /var/log/gitlab/registry/current
2020-01-21_13:46:16.49320 time="2020-01-21T14:46:16.493118369+01:00" level=warning msg="error authorizing context: authorization token required" go.version=go1.12.13 http.request.host="gitlab.mydomain.com:5050" http.request.id=fbe88f1e-ccf5-4fcd-8f3a-aa03d216388a http.request.method=GET http.request.remoteaddr=8.8.8.8 http.request.uri="/v2/" http.request.useragent="docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \(linux\))"
2020-01-21_13:46:16.49351 127.0.0.1 - - [21/Jan/2020:14:46:16 +0100] "GET /v2/ HTTP/1.1" 401 87 "" "docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \\(linux\\))"
2020-01-21_13:46:17.10631 time="2020-01-21T14:46:17.10627187+01:00" level=info msg="authorized request" go.version=go1.12.13 http.request.host="gitlab.mydomain.com:5050" http.request.id=7cc76f13-b5f3-4f4d-9309-d338b9c5c8b5 http.request.method=HEAD http.request.remoteaddr=8.8.8.8 http.request.uri="/v2/testing/registry/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" http.request.useragent="docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \(linux\))" vars.digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" vars.name="testing/registry"
2020-01-21_13:46:17.10687 time="2020-01-21T14:46:17.106817596+01:00" level=error msg="response completed with error" auth.user.name=myname err.code=unknown err.detail="filesystem: open /mnt/data/git-data/gitlab-rails/shared/registry/docker/registry/v2/repositories/testing/registry/_layers/sha256/c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9/link: permission denied" err.message="unknown error" go.version=go1.12.13 http.request.host="gitlab.mydomain.com:5050" http.request.id=7cc76f13-b5f3-4f4d-9309-d338b9c5c8b5 http.request.method=HEAD http.request.remoteaddr=8.8.8.8 http.request.uri="/v2/testing/registry/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" http.request.useragent="docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \(linux\))" http.response.contenttype="application/json; charset=utf-8" http.response.duration=2.192904ms http.response.status=500 http.response.written=320 vars.digest="sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9" vars.name="testing/registry"
2020-01-21_13:46:17.10702 127.0.0.1 - - [21/Jan/2020:14:46:17 +0100] "HEAD /v2/testing/registry/blobs/sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9 HTTP/1.1" 500 320 "" "docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \\(linux\\))"
2020-01-21_13:46:17.16482 time="2020-01-21T14:46:17.164783711+01:00" level=info msg="authorized request" go.version=go1.12.13 http.request.host="gitlab.mydomain.com:5050" http.request.id=e3e752c1-442a-46b1-b7c4-3f997e6e97a6 http.request.method=POST http.request.remoteaddr=8.8.8.8 http.request.uri="/v2/testing/registry/blobs/uploads/" http.request.useragent="docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \(linux\))" vars.name="testing/registry"
2020-01-21_13:46:17.16537 time="2020-01-21T14:46:17.165324403+01:00" level=error msg="response completed with error" auth.user.name=myname err.code=unknown err.detail="filesystem: mkdir /mnt/data/git-data/gitlab-rails: permission denied" err.message="unknown error" go.version=go1.12.13 http.request.host="gitlab.mydomain.com:5050" http.request.id=e3e752c1-442a-46b1-b7c4-3f997e6e97a6 http.request.method=POST http.request.remoteaddr=8.8.8.8 http.request.uri="/v2/testing/registry/blobs/uploads/" http.request.useragent="docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \(linux\))" http.response.contenttype="application/json; charset=utf-8" http.response.duration=2.673484ms http.response.status=500 http.response.written=171 vars.name="testing/registry"
2020-01-21_13:46:17.16554 127.0.0.1 - - [21/Jan/2020:14:46:17 +0100] "POST /v2/testing/registry/blobs/uploads/ HTTP/1.1" 500 171 "" "docker/19.03.5 go/go1.12.12 git-commit/633a0ea838 kernel/4.19.0-6-amd64 os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.5 \\(linux\\))"
Самая большая проблема (если я правильно понимаю) это:
filesystem: open /mnt/data/git-data/gitlab-rails/shared/registry/docker/registry/v2/repositories/testing/registry/_layers/sha256/c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9/link: permission denied
filesystem: mkdir /mnt/data/git-data/gitlab-rails: permission denied
Содержание каталога
registryНа пути нет ни одного директора
repo:/# ll /mnt/data/git-data/gitlab-rails/shared/
total 40
drwxr-x--x 10 git gitlab-www 4096 Jan 21 14:11 .
drwxr-xr-x 3 root root 4096 Sep 24 2018 ..
drwx------ 11 git root 4096 Dec 10 08:21 artifacts
drwx------ 3 git root 4096 Oct 24 2018 cache
drwx------ 2 git root 4096 Jul 30 10:36 dependency_proxy
drwx------ 2 git root 4096 Jul 30 10:36 external-diffs
drwx------ 259 git root 4096 Oct 25 2018 lfs-objects
drwx------ 2 git root 4096 Dec 3 2018 packages
drwxr-x--- 9 git gitlab-www 4096 Dec 10 09:12 pages
drwx------ 3 git root 4096 Sep 24 2018 tmp
Конфигурация Gitlab
root@repo:gitlab-ctl show-config
Starting Chef Client, version 14.13.11
resolving cookbooks for run list: ["gitlab::show_config"]
Synchronizing Cookbooks:
- redis (0.1.0)
- registry (0.1.0)
- gitaly (0.1.0)
- letsencrypt (0.1.0)
- gitlab (0.0.1)
- runit (4.3.0)
- crond (0.1.0)
- package (0.1.0)
- postgresql (0.1.0)
- consul (0.1.0)
- nginx (0.1.0)
- mattermost (0.1.0)
- acme (4.0.0)
- praefect (0.1.0)
- monitoring (0.1.0)
Installing Cookbook Gems:
Compiling Cookbooks...
{
"gitlab": {
"gitlab-shell": {
"secret_token": "<some_hash>",
"auth_file": "/var/opt/gitlab/.ssh/authorized_keys"
},
"gitlab-rails": {
"lfs_enabled": true,
"lfs_storage_path": "/mnt/data/git-data/gitlab-rails/shared/lfs-objects",
"backup_path": "/mnt/data/gitlab-backup/",
"backup_keep_time": 604800,
"shared_path": "/mnt/data/git-data/gitlab-rails/shared",
"secret_key_base": "<some_hash>",
"db_key_base": "<some_hash>",
"otp_key_base": "<some_hash>",
"openid_connect_signing_key": "-----BEGIN RSA PRIVATE KEY-----\n<some_hash>\n-----END RSA PRIVATE KEY-----\n",
"gitlab_host": "gitlab.mydomain.com",
"gitlab_email_from": "[email protected]",
"gitlab_https": true,
"gitlab_port": 443,
"artifacts_path": "/mnt/data/git-data/gitlab-rails/shared/artifacts",
"external_diffs_storage_path": "/mnt/data/git-data/gitlab-rails/shared/external-diffs",
"uploads_storage_path": "/opt/gitlab/embedded/service/gitlab-rails/public",
"packages_storage_path": "/mnt/data/git-data/gitlab-rails/shared/packages",
"dependency_proxy_storage_path": "/mnt/data/git-data/gitlab-rails/shared/dependency_proxy",
"pages_path": "/mnt/data/git-data/gitlab-rails/shared/pages",
"repositories_storages": {
"default": {
"path": "/mnt/data/git-data/repositories",
"gitaly_address": "unix:/var/opt/gitlab/gitaly/gitaly.socket"
}
},
"trusted_proxies": [
],
"db_username": "gitlab",
"db_host": null,
"db_port": 5432
},
"gitlab-workhorse": {
"secret_token": "<some_hash>",
"auth_socket": "/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"
},
"logging": {
},
"unicorn": {
},
"puma": {
},
"mailroom": {
},
"gitlab-pages": {
"gitlab_secret": null,
"gitlab_id": null,
"auth_secret": "<some_hash>",
"api_secret_key": "<some_hash>"
},
"external-url": "https://gitlab.mydomain.com",
"registry-external-url": null,
"mattermost-external-url": null,
"pages-external-url": null,
"runtime-dir": "/run",
"git-data-dir": null,
"bootstrap": {
},
"omnibus-gitconfig": {
},
"manage-accounts": {
},
"manage-storage-directories": {
},
"user": {
"home": "/var/opt/gitlab",
"git_user_email": "[email protected]"
},
"gitlab-ci": {
},
"sidekiq": {
},
"mattermost-nginx": {
"listen_port": null
},
"pages-nginx": {
"listen_port": null
},
"registry-nginx": {
},
"remote-syslog": {
},
"logrotate": {
},
"high-availability": {
},
"web-server": {
},
"prometheus-monitoring": {
},
"pgbouncer": {
},
"pgbouncer-exporter": {
},
"storage-check": {
"target": "unix:///var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"
},
"nginx": {
"redirect_http_to_https": true,
"ssl_certificate": "/etc/gitlab/ssl/gitlab.mydomain.com.crt",
"ssl_certificate_key": "/etc/gitlab/ssl/gitlab.mydomain.com.key",
"proxy_set_headers": {
"Host": "$http_host_with_default",
"X-Real-IP": "$remote_addr",
"X-Forwarded-For": "$proxy_add_x_forwarded_for",
"Upgrade": "$http_upgrade",
"Connection": "$connection_upgrade",
"X-Forwarded-Proto": "https",
"X-Forwarded-Ssl": "on"
},
"real_ip_trusted_addresses": [
],
"listen_port": 443
}
},
"roles": {
"application": {
},
"redis-sentinel": {
},
"redis-master": {
},
"redis-slave": {
},
"geo-primary": {
},
"geo-secondary": {
},
"monitoring": {
},
"postgres": {
},
"pgbouncer": {
},
"consul": {
}
},
"monitoring": {
"prometheus": {
"alertmanagers": [
],
"flags": {
"web.listen-address": "localhost:9090",
"storage.tsdb.path": "/var/opt/gitlab/prometheus/data",
"config.file": "/var/opt/gitlab/prometheus/prometheus.yml"
}
},
"grafana": {
"secret_key": "7dfc8ff446078cdabd489b77ec25fa37",
"gitlab_secret": "<some_hash>",
"gitlab_application_id": "<some_hash>",
"admin_password": "<some_hash>",
"metrics_basic_auth_password": null,
"datasources": [
{
"name": "GitLab Omnibus",
"type": "prometheus",
"access": "proxy",
"url": "http://localhost:9090",
"isDefault": true
}
]
},
"alertmanager": {
"flags": {
"web.listen-address": "localhost:9093",
"storage.path": "/var/opt/gitlab/alertmanager/data",
"config.file": "/var/opt/gitlab/alertmanager/alertmanager.yml"
}
},
"node-exporter": {
"flags": {
"web.listen-address": "localhost:9100",
"collector.mountstats": true,
"collector.runit": true,
"collector.runit.servicedir": "/opt/gitlab/sv",
"collector.textfile.directory": "/var/opt/gitlab/node-exporter/textfile_collector"
}
},
"redis-exporter": {
"flags": {
"web.listen-address": "localhost:9121",
"redis.addr": "unix:///var/opt/gitlab/redis/redis.socket"
}
},
"postgres-exporter": {
"flags": {
"web.listen-address": "localhost:9187",
"extend.query-path": "/var/opt/gitlab/postgres-exporter/queries.yaml"
}
},
"gitlab-exporter": {
"probe_sidekiq": true
},
"gitlab-monitor": {
}
},
"letsencrypt": {
"auto_enabled": false,
"enable": false
},
"package": {
},
"registry": {
"health_storagedriver_enabled": false,
"http_secret": "<some_hash>",
"internal_certificate": "-----BEGIN CERTIFICATE-----\<some_hash>\n-----END CERTIFICATE-----\n",
"internal_key": "-----BEGIN RSA PRIVATE KEY-----\n<some_hash>\n-----END RSA PRIVATE KEY-----\n"
},
"redis": {
"rename_commands": {
"KEYS": ""
}
},
"postgresql": {
"internal_certificate": "-----BEGIN CERTIFICATE-----\n<some_hash>\n-----END CERTIFICATE-----\n",
"internal_key": "-----BEGIN RSA PRIVATE KEY-----\n<some_hash>\n-----END RSA PRIVATE KEY-----\n"
},
"repmgr": {
},
"repmgrd": {
},
"consul": {
},
"gitaly": {
"storage": [
{
"name": "default",
"path": "/mnt/data/git-data/repositories"
}
]
},
"praefect": {
},
"crond": {
},
"mattermost": {
"email_invite_salt": "<some_hash>",
"file_public_link_salt": "<some_hash>",
"sql_at_rest_encrypt_key": "<some_hash>",
"sql_data_source": "user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production"
}
}
Converging 0 resources
Running handlers:
Running handlers complete
Chef Client finished, 0/0 resources updated in 06 seconds
Информация об окружении GitLab
repo:/# gitlab-rake gitlab:env:info
System information
System: Debian 8.11
Proxy: no
Current User: git
Using RVM: no
Ruby Version: 2.6.3p62
Gem Version: 2.7.9
Bundler Version:1.17.3
Rake Version: 12.3.3
Redis Version: 3.2.12
Git Version: 2.24.1
Sidekiq Version:5.2.7
Go Version: unknown
GitLab information
Version: 12.6.4-ee
Revision: cc6b787e7b0
Directory: /opt/gitlab/embedded/service/gitlab-rails
DB Adapter: PostgreSQL
DB Version: 10.9
URL: https://gitlab.mydomain.com
HTTP Clone URL: https://gitlab.mydomain.com/some-group/some-project.git
SSH Clone URL: [email protected]:some-group/some-project.git
Elasticsearch: no
Geo: no
Using LDAP: no
Using Omniauth: yes
Omniauth Providers:
GitLab Shell
Version: 10.3.0
Repository storage paths:
- default: /mnt/data/git-data/repositories
GitLab Shell path: /opt/gitlab/embedded/service/gitlab-shell
Git: /opt/gitlab/embedded/bin/git
решение1
Кто является владельцем вашего каталога реестра?
Попробуйте изменить владельца на "registry". У меня была похожая проблема, и я изменил владельца "/var/opt/gitlab/gitlab-rails/shared/registry/docker/registry/" с "git" на "registry"
решение2
"/var/opt/gitlab/gitlab-rails/shared/registry/docker/registry/" с "git" на "registry" сработало для меня, появилось после обновления Gitlab с 15-го на 14-е