"fields.time": "2019-05-24T07:48:03Z",
"latency": 373642,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50812",
"request": "/",
"request-id": "1LffTyeEQ2po3jqYwJkMC1sBuL8",
"time": "2019-05-24T07:48:03Z"
}
{
"fields.time": "2019-05-24T07:48:04Z",
"latency": 502729,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50812",
"request": "/css/app.380775a8.css",
"request-id": "1LffU3U76prsmLUbIbLe0BZUHCg",
"time": "2019-05-24T07:48:04Z"
}
{
"level": "debug",
"msg": "api: authentication required",
"request-id": "1LffU3cgnEOqknBab4GspWKtOHf",
"time": "2019-05-24T07:48:04Z"
}
{
"level": "debug",
"msg": "api: guest access",
"request-id": "1LffU3cgnEOqknBab4GspWKtOHf",
"time": "2019-05-24T07:48:04Z"
}
{
"fields.time": "2019-05-24T07:48:04Z",
"latency": 482892,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50818",
"request": "/api/user",
"request-id": "1LffU3cgnEOqknBab4GspWKtOHf",
"time": "2019-05-24T07:48:04Z"
}
{
"fields.time": "2019-05-24T07:48:04Z",
"latency": 21593,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50818",
"request": "/favicon.png",
"request-id": "1LffU4pTVmkk74H0VtJa2fEtLLQ",
"time": "2019-05-24T07:48:04Z"
}
{
"level": "debug",
"msg": "events: stream opened",
"request-id": "1LffU7q8ayyQahLfWDPsc6dOAp8",
"time": "2019-05-24T07:48:04Z"
}
{
"fields.time": "2019-05-24T07:48:04Z",
"latency": 41482,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50822",
"request": "/login",
"request-id": "1LffU4lvDyLOQKa2FNwRTbdEBkI",
"time": "2019-05-24T07:48:04Z"
}
{
"level": "debug",
"login": "mshevtsov",
"msg": "attempting authentication",
"time": "2019-05-24T07:48:05Z"
}
{
"level": "debug",
"login": "mshevtsov",
"msg": "authentication successful",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 843148417,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50822",
"request": "/login?code=5930c41fe4da5e62c07fe09e168657db2d6eb48ac9da6e6a549fc5045ea6cae3\u0026state=4d65822107fcfd52",
"request-id": "1LffU4PECIToe0zq16SWDmpkA6i",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 617378,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50822",
"request": "/",
"request-id": "1LffUE00ppuLpYa3qV1qOtfH7Xu",
"time": "2019-05-24T07:48:05Z"
}
{
"level": "debug",
"msg": "events: stream eror",
"request-id": "1LffU7q8ayyQahLfWDPsc6dOAp8",
"time": "2019-05-24T07:48:05Z"
}
{
"level": "debug",
"msg": "events: stream closed",
"request-id": "1LffU7q8ayyQahLfWDPsc6dOAp8",
"time": "2019-05-24T07:48:05Z"
}
{
"level": "debug",
"msg": "api: guest access",
"request-id": "1LffU7q8ayyQahLfWDPsc6dOAp8",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 1087202529,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50818",
"request": "/api/stream",
"request-id": "1LffU7q8ayyQahLfWDPsc6dOAp8",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 164171,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50822",
"request": "/api/user",
"request-id": "1LffUAU9XRCgoypGgzU50YkUsRB",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 33136,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50822",
"request": "/favicon.png",
"request-id": "1LffUCrLxdI5meNVwmNLucdjnuh",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 1416757,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50832",
"request": "/api/user/repos?latest=true",
"request-id": "1LffUCwJSzT4MNAcorNqMAXytUT",
"time": "2019-05-24T07:48:05Z"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 409337,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50832",
"request": "/api/user/builds/recent",
"request-id": "1LffUCgaw7xVmReQpTHXXgZSnh3",
"time": "2019-05-24T07:48:05Z"
}
{
"level": "debug",
"msg": "events: stream opened",
"request-id": "1LffUEfTsLXUyGgKjJQaCIHu8sC",
"time": "2019-05-24T07:48:05Z",
"user.login": "mshevtsov"
}
{
"fields.time": "2019-05-24T07:48:05Z",
"latency": 1372182,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50832",
"request": "/api/user/repos?latest=true",
"request-id": "1LffUEl3BOHVuiGKZm3DPzvYaes",
"time": "2019-05-24T07:48:05Z"
}
{
"admin": false,
"level": "debug",
"msg": "api: sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV6bwOluLqAmbQ6MwfiZa8m3",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"admin": false,
"level": "debug",
"msg": "api: sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV5x5rKgmfmRTXFdqkM8SU08",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"admin": false,
"level": "debug",
"msg": "api: sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV5OKPbspuNwXtrMV5HaywDN",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"admin": false,
"error": "",
"level": "warning",
"msg": "api: cannot sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV6bwOluLqAmbQ6MwfiZa8m3",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"fields.time": "2019-05-24T07:48:12Z",
"latency": 159149443,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50832",
"request": "/api/repos/mshevtsov/hello_hapi",
"request-id": "1LffV6bwOluLqAmbQ6MwfiZa8m3",
"time": "2019-05-24T07:48:12Z"
}
{
"admin": false,
"error": "",
"level": "warning",
"msg": "api: cannot sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV5x5rKgmfmRTXFdqkM8SU08",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"fields.time": "2019-05-24T07:48:12Z",
"latency": 231966822,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50840",
"request": "/api/repos/mshevtsov/hello_hapi/secrets",
"request-id": "1LffV5x5rKgmfmRTXFdqkM8SU08",
"time": "2019-05-24T07:48:12Z"
}
{
"admin": false,
"error": "",
"level": "warning",
"msg": "api: cannot sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV5OKPbspuNwXtrMV5HaywDN",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
"write": false
}
{
"fields.time": "2019-05-24T07:48:12Z",
"latency": 276594871,
"level": "debug",
"method": "GET",
"msg": "",
"remote": "172.17.0.1:50844",
"request": "/api/repos/mshevtsov/hello_hapi/cron",
"request-id": "1LffV5OKPbspuNwXtrMV5HaywDN",
"time": "2019-05-24T07:48:12Z"
}
one thing I see is that your account does not have write
or admin
access to the repository. Admin access is required in order to activate a repository.
{
+ "admin": false,
"level": "debug",
"msg": "api: sync repository permissions",
"name": "hello_hapi",
"namespace": "mshevtsov",
"read": true,
"request-id": "1LffV6bwOluLqAmbQ6MwfiZa8m3",
"time": "2019-05-24T07:48:12Z",
"user.login": "mshevtsov",
+ "write": false
}
the second thing I notice is, per the gitlab server logs you posted, that gitlab is returning a 404 not found when drone tries to query the repository to verify your account’s access level:
192.168.1.211 - - [23/May/2019:22:51:00 +0300] "GET /api/v4/projects/mshevtsov/hello_hapi HTTP/1.0" 404 25 "" "Go-http-client/1.1"
so if gitlab returns a 404 not found when trying to access the repository, it is expected that drone will not be able to sync permissions to a repository that gitlab says does not exist.
So. When I launch drone/drone:1.1 via docker-compose I get errors reported above (only guest access). When I use docker run, with a gitlab server on prem and a root user that has access to everything and all permissions are wide open, it gets past the oAuth and initial repo synchronization. When I try to activate I get “api: cannot create or update hook”.
{"admin":true,"level":"debug","msg":"api: repository permissions synchronized","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDMut37tTMoZ0dLgOZcWSjxy","time":"2019-05-27T13:28:10Z","user.login":"root","write":true}
{"level":"debug","msg":"api: read access granted","name":"drone-artifactory","namespace":"pci","request-id":"1LooDMut37tTMoZ0dLgOZcWSjxy","time":"2019-05-27T13:28:10Z","user.login":"root","visibility":"public"}
{"fields.time":"2019-05-27T13:28:10Z","latency":74203586,"level":"debug","method":"GET","msg":"","remote":"67.165.20.225:56863","request":"/api/repos/pci/drone-artifactory","request-id":"1LooDMut37tTMoZ0dLgOZcWSjxy","time":"2019-05-27T13:28:10Z"}
{"admin":true,"level":"debug","msg":"api: repository permissions synchronized","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDMO3wI4tSFZqwjexq6tpKur","time":"2019-05-27T13:28:10Z","user.login":"root","write":true}
{"level":"debug","msg":"api: read access granted","name":"drone-artifactory","namespace":"pci","request-id":"1LooDMO3wI4tSFZqwjexq6tpKur","time":"2019-05-27T13:28:10Z","user.login":"root","visibility":"public"}
{"admin":true,"level":"debug","msg":"api: access granted","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDMO3wI4tSFZqwjexq6tpKur","time":"2019-05-27T13:28:10Z","user.login":"root","visibility":"public","write":true}
{"fields.time":"2019-05-27T13:28:10Z","latency":56141513,"level":"debug","method":"GET","msg":"","remote":"67.165.20.225:56868","request":"/api/repos/pci/drone-artifactory/cron","request-id":"1LooDMO3wI4tSFZqwjexq6tpKur","time":"2019-05-27T13:28:10Z"}
{"admin":true,"level":"debug","msg":"api: repository permissions synchronized","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDPGHlwdhwS0wOZ1dviFUdX1","time":"2019-05-27T13:28:10Z","user.login":"root","write":true}
{"level":"debug","msg":"api: read access granted","name":"drone-artifactory","namespace":"pci","request-id":"1LooDPGHlwdhwS0wOZ1dviFUdX1","time":"2019-05-27T13:28:10Z","user.login":"root","visibility":"public"}
{"admin":true,"level":"debug","msg":"api: access granted","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDPGHlwdhwS0wOZ1dviFUdX1","time":"2019-05-27T13:28:10Z","user.login":"root","visibility":"public","write":true}
{"fields.time":"2019-05-27T13:28:10Z","latency":69322511,"level":"debug","method":"GET","msg":"","remote":"67.165.20.225:56869","request":"/api/repos/pci/drone-artifactory/secrets","request-id":"1LooDPGHlwdhwS0wOZ1dviFUdX1","time":"2019-05-27T13:28:10Z"}
{"level":"debug","msg":"api: read access granted","name":"drone-artifactory","namespace":"pci","request-id":"1LooDiBXJhsUXv0pYok6rVN13Vb","time":"2019-05-27T13:28:12Z","user.login":"root","visibility":"public"}
{"admin":true,"level":"debug","msg":"api: access granted","name":"drone-artifactory","namespace":"pci","read":true,"request-id":"1LooDiBXJhsUXv0pYok6rVN13Vb","time":"2019-05-27T13:28:12Z","user.login":"root","visibility":"public","write":true}
{"error":"","level":"debug","msg":"api: cannot create or update hook","name":"drone-artifactory","namespace":"pci","request-id":"1LooDiBXJhsUXv0pYok6rVN13Vb","time":"2019-05-27T13:28:12Z","user.login":"root"}
- I’ve checked gitlab drone application scopes and made sure they are: api (Access the authenticated user’s API), read_user (Read the authenticated user’s personal information).
- I’ve updated gitlab to 11.11.0 (Latest version).
- I did check 3 times that I’m admin in gitlab. Also migrated repository from my group to another (called
internal
) where I’mowner
but I still get those errors:
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrF9BzEi6bHqICa1tEzKGAf88e","time":"2019-05-28T10:09:15Z","user.login":"mshevtsov","write":false}
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrF9GCfppMJ1A8zN8BfCbvzUJb","time":"2019-05-28T10:09:15Z","user.login":"mshevtsov","write":false}
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrF9DPoK4H1dWHu3f9kpCljxtz","time":"2019-05-28T10:09:15Z","user.login":"mshevtsov","write":false}
- Even If I login as
root
still getting same error:
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrHYNdxaLswAbF4uHEgKoZpNQK","time":"2019-05-28T10:29:03Z","user.admin":true,"user.login":"root","write":false}
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrHYLl40f7JGQPAwx7H5cbKt1Q","time":"2019-05-28T10:29:03Z","user.admin":true,"user.login":"root","write":false}
{"admin":false,"error":"","level":"warning","msg":"api: cannot sync repository permissions","name":"hello_hapi","namespace":"internal","read":true,"request-id":"1LrHYR8vk3sFsUKhVn6VAf6c3ju","time":"2019-05-28T10:29:03Z","user.admin":true,"user.login":"root","write":false}
What I’m doing wrong?
@chrispauleypci I do not have a private gitlab instance, but I tested with gitlab.com to confirm that I am able to activate repositories without issue. I also verified with some of our existing customers using GitLab. This leads me to believe we are dealing with a configuration or environment-specific issue.
{
"admin": true,
"level": "debug",
"msg": "api: access granted",
"name": "hello-world",
"namespace": "bradrydzewski",
"read": true,
"request-id": "1LpADNydReAspSYoZgA0b6WRNrS",
"time": "2019-05-27T09:29:04-07:00",
"user.login": "bradrydzewski",
"visibility": "private",
"write": true
}
{
"fields.time": "2019-05-27T09:29:05-07:00",
"latency": 1445153577,
"level": "debug",
"method": "POST",
"msg": "",
"remote": "[::1]:51869",
"request": "/api/repos/bradrydzewski/hello-world",
"request-id": "1LpADNydReAspSYoZgA0b6WRNrS",
"time": "2019-05-27T09:29:05-07:00"
}
The error you posted in your logs comes from the following block of code:
err = hooks.Create(r.Context(), user, repo)
if err != nil {
render.InternalError(w, err)
logger.FromRequest(r).
WithError(err).
WithField("namespace", owner).
WithField("name", name).
Debugln("api: cannot create or update hook")
return
}
The good news is only one thing happens in this block of code — Drone makes an http request to GitLab to create the webhook. This really narrows down the list of possible root causes.
Some common root causes we see:
- User activating the repository is not a repository administrator. Are you able to create webhooks in the GitLab user interface for this repository? If no, you need to work with the repository administrator to get admin access. Being a Drone administrator alone does not satisfy this requirement.
- Automatic redirect by GitLab or reverse proxy (converts a POST request to a GET request) thus causing the request to fail. Or some other reverse proxy issues. I doubt this is the root cause since it would likely have impacted oauth login and redirect, but still worth noting.
- Invalid webhook is generated. This usually happens when you configure an invalid server host and protocol. I doubt this is the root cause since it would likely have impacted oauth login and redirect, but still worth noting.
- Invalid or missing scopes when you created the oauth application. The list of required scopes are
api
andread_user
I also noticed that your GitLab server address is not a valid URL (missing http:// or https:// scheme). I am a bit surprised you were able to get this far without specifying the scheme, which leads me to believe this is not the root cause. Although worth exploring.
I think the next step would be to look at your GitLab server logs to see why the request is failing. With these logs we should be able to diagnose the exact root cause.
@mesouug per my previous comment, GitLab is returning a 404 not found for the repository when making an API request using your oauth2 token. At this point, this is a GitLab issue, not a Drone issue. If GitLab returns a 404 and tells Drone a repository is not found, there is nothing that Drone can do. You need to work with GitLab support to understand why a 404 is being returned. Once you have a root cause to share we can resume the conversation.
192.168.1.211 - - [23/May/2019:22:51:00 +0300] "GET /api/v4/projects/mshevtsov/hello_hapi HTTP/1.0" 404 25 "" "Go-http-client/1.1"
@bradrydzewski,
So I changed the drone/drone1.1 docker-compose env variable DRONE_GITLAB_SERVER to use http:// as the scheme (running on an ec2 host), pointed it at our live gitlab on prem server and the post to drone worked (201 response). Great. Except we cannot use a pipeline that uses a sandbox instance that points to our dev server.
When I point another drone/drone1.1 server at a gitlab server sameersbn/gitlab:11.10.4 on the same ec2 host, the gitlab server logged a 422 error when gitlab posted the webhook to drone. /var/log/gitlab/gitlab/api_json.log
{"time":"2019-05-28T17:03:15.004Z","severity":"INFO","duration":11.54,"db":2.69,"view":8.85,"status":200,"method":"GET","path":"/api/v4/projects/pci%2Fdrone-artifactory/hooks","params":[{"key":"per_page","value":"100"}],"host":"redacted.com
pute-1.amazonaws.com","ip":"172.19.0.1, 127.0.0.1","ua":"Go-http-client/1.1","route":"/api/:version/projects/:id/hooks","user_id":1,"username":"root","queue_duration":46.25,"gitaly_calls":0,"gitaly_duration":0,"correlation_id":"MbyagTlF5M5"}
{"time":"2019-05-28T17:03:15.024Z","severity":"INFO","duration":13.61,"db":1.98,"view":11.629999999999999,"status":422,"method":"POST","path":"/api/v4/projects/pci%2Fdrone-artifactory/hooks","params":[{"key":"merge_requests_events","value":"true"
},{"key":"push_events","value":"true"},{"key":"tag_push_events","value":"true"},{"key":"token","value":"[FILTERED]"},{"key":"url","value":"http://redacted.compute-1.amazonaws.com:8092/hook"}],"host":"redacted.compute-1.amazonaws.com",
"ip":"172.19.0.1, 127.0.0.1","ua":"Go-http-client/1.1","route":"/api/:version/projects/:id/hooks","user_id":1,"username":"root","queue_duration":5.23,"gitaly_calls":0,"gitaly_duration":0,"correlation_id":"IFSTPslSE2"}
@chrispauleypci do you need to enable some setting to allow gitlab to post webhooks to an address that resolves to the same servers as gitlab itself? It looks like you might find some settings here: https://gitlab.com/gitlab-org/gitlab-ce/issues/55605
or maybe alternatively, run them on separate hosts in your test environment?
@bradrydzewski The link you provided resolved the issue of running gitlab and the drone server on the ec2 host. Allow allow_local_requests_from_hooks_and_services
to be set via API fixed that http 422 issue, thanks. We can update our drone .4 scripts to v1. Now I need to get this to work behind a reverse proxy on the development host server. Thanks again!
As far as I remember Gitlab CE always used following URI for accessing project info: /api/v4/projects/59
instead of /api/v4/projects/mshevtsov/hello_hapi
.
I was thinking if this is my setup issue but I’ve tried to open any project via /api/v4/...
on https://gitlab.com but it’s also returns 404. And documentation clearly says that we should use project ID instead of name.
Maybe this is a bug?
Thanks,
Mikhail.
The documentation clearly defines the :id
as a The ID or URL-encoded path of the project
[1]. There are many members of our community successfully using Drone + GitLab without issue, and I have personally tested to confirm that it is working correctly. I therefore recommend you continue working with GitLab support to troubleshoot the 404 error.
[1] https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
Thank you very much for your hints.
I finally found root cause of my problem. It was indeed my setup. My gitlab sits behind nginx reverse proxy and proxy_pass https://192.168.1.220/
was converting %2F
to /
once I removed trailing slash in proxy_pass https://192.168.1.220
everything started to work as expected. No wonder I couldn’t make it work.
Now in gitlab logs everything is good:
{"time":"2019-05-29T15:23:14.868Z","severity":"INFO","duration":142.31,"db":52.02,"view":90.28999999999999,"status":200,"method":"GET","path":"/api/v4/projects/internal%2Fhello_hapi","params":[],"host":"git.mesouug.com","ip":"188.162.185.135, 192.168.1.211, 188.162.185.135","ua":"Go-http-client/1.1","route":"/api/:version/projects/:id","user_id":2,"username":"mshevtsov","queue_duration":22.07,"gitaly_calls":0,"gitaly_duration":0,"correlation_id":"yqlZcp5v5C6"}
Thank you again for your assistance!
great, glad you were able to get it working!
Howdy folks,
Just posting here since I was running into the same issue. It looks like requests need to be allowed to 3rd party services (see https://gitlab.com/gitlab-org/gitlab-foss/issues/55605#note_126398368).
@bradrydzewski is this something that Drone might be able to surface in your all’s docs as part of the installation docs?