diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 2456949..124ed69 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -19,6 +19,9 @@ gitlab-template { // Pagination value for Gitlab. See https://docs.gitlab.com/ee/api/README.html#pagination for details. per-page: 100 + + // throttle interval (milliseconds) + throttle-duration: 100 } technical-users-keys { diff --git a/src/main/scala/de/frosner/gitlabtemplate/GitlabSource.scala b/src/main/scala/de/frosner/gitlabtemplate/GitlabSource.scala index 689d053..bf15a75 100644 --- a/src/main/scala/de/frosner/gitlabtemplate/GitlabSource.scala +++ b/src/main/scala/de/frosner/gitlabtemplate/GitlabSource.scala @@ -34,7 +34,8 @@ class GitlabSource(wsClient: StandaloneWSClient, url: String, privateToken: String, onlyActiveUsers: Boolean, - perPage: Int)(implicit ec: ExecutionContext, materializer: Materializer) + perPage: Int, + throttleDuration: Long)(implicit ec: ExecutionContext, materializer: Materializer) extends StrictLogging { def getUsers: EitherT[Future, Error, Set[GitlabUser]] = { @@ -67,6 +68,7 @@ class GitlabSource(wsClient: StandaloneWSClient, .url(s"$url/api/v4/users/${user.id}/keys") .withHttpHeaders(("PRIVATE-TOKEN", privateToken)) logger.debug(s"Requesting public keys for ${user.username}: ${request.url}") + Thread.sleep(throttleDuration) request.get().map { response => Json .fromJson[Set[PublicKey]](response.body[JsValue]) diff --git a/src/main/scala/de/frosner/gitlabtemplate/GitlabTemplateConfig.scala b/src/main/scala/de/frosner/gitlabtemplate/GitlabTemplateConfig.scala index d52a967..c6372d8 100644 --- a/src/main/scala/de/frosner/gitlabtemplate/GitlabTemplateConfig.scala +++ b/src/main/scala/de/frosner/gitlabtemplate/GitlabTemplateConfig.scala @@ -22,6 +22,6 @@ case class PrivateTokenAuthConfig(enabled: Boolean, token: String) case class SinkConfig(filesystem: FilesystemConfig) -case class GitlabConfig(onlyActiveUsers: Boolean, privateToken: String, url: String, perPage: Int) +case class GitlabConfig(onlyActiveUsers: Boolean, privateToken: String, url: String, perPage: Int, throttleDuration: Long) case class FilesystemConfig(path: Path, publicKeysFile: String, createEmptyKeyFile: Boolean) diff --git a/src/main/scala/de/frosner/gitlabtemplate/Main.scala b/src/main/scala/de/frosner/gitlabtemplate/Main.scala index 67fc27d..49030a2 100644 --- a/src/main/scala/de/frosner/gitlabtemplate/Main.scala +++ b/src/main/scala/de/frosner/gitlabtemplate/Main.scala @@ -51,7 +51,8 @@ object Main extends StrictLogging { gitlabConf.url, gitlabConf.privateToken, gitlabConf.onlyActiveUsers, - gitlabConf.perPage) + gitlabConf.perPage, + gitlabConf.throttleDuration) val technicalUsersKeysSource = new TechnicalUsersKeysSource(wsClient, diff --git a/src/test/scala/de/frosner/gitlabtemplate/GitlabSourceSpec.scala b/src/test/scala/de/frosner/gitlabtemplate/GitlabSourceSpec.scala index 701bbad..a810020 100644 --- a/src/test/scala/de/frosner/gitlabtemplate/GitlabSourceSpec.scala +++ b/src/test/scala/de/frosner/gitlabtemplate/GitlabSourceSpec.scala @@ -48,7 +48,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { GitlabUser(1, "usr1"), GitlabUser(2, "usr2") ) - new GitlabSource(wsClient, address, "token", false, 100).getUsers.value + new GitlabSource(wsClient, address, "token", false, 100, 1).getUsers.value .map(_ shouldBe Right(expected)) } } @@ -76,7 +76,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { GitlabUser(6, "usr6"), GitlabUser(7, "usr7") ) - new GitlabSource(wsClient, address, "token", false, 2).getUsers.value + new GitlabSource(wsClient, address, "token", false, 2, 1).getUsers.value .map(_ shouldBe Right(expected)) } } @@ -97,7 +97,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { } { implicit ec => implicit materializer => { case (wsClient, address) => - new GitlabSource(wsClient, address, "token", false, 100).getUsers.value + new GitlabSource(wsClient, address, "token", false, 100, 1).getUsers.value .map(_ shouldBe a[Left[_, _]]) } } @@ -107,7 +107,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { withServerAndClient { RouteDirectives.reject } { implicit ec => implicit materializer => { case (wsClient, address) => - new GitlabSource(wsClient, "http://dsafdsgdfsfdsfdsf", "token", false, 100).getUsers.value.failed + new GitlabSource(wsClient, "http://dsafdsgdfsfdsfdsf", "token", false, 100, 1).getUsers.value.failed .map(_ shouldBe a[UnknownHostException]) } } @@ -135,7 +135,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { "usr1" -> Set("key1", "key2"), "usr2" -> Set("key3") ) - new GitlabSource(wsClient, address, "token", false, 100) + new GitlabSource(wsClient, address, "token", false, 100, 1) .getSshKeys(users) .value .map(_ shouldBe Right(expected)) @@ -153,7 +153,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { } { implicit ec => implicit materializer => { case (wsClient, address) => - new GitlabSource(wsClient, address, "token", false, 100) + new GitlabSource(wsClient, address, "token", false, 100, 1) .getSshKeys(Set(GitlabUser(1, "usr1"))) .value .map(_ shouldBe a[Left[_, _]]) @@ -165,7 +165,7 @@ class GitlabSourceSpec extends FlatSpec with Matchers with HttpTests { withServerAndClient { RouteDirectives.reject } { implicit ec => implicit materializer => { case (wsClient, address) => - new GitlabSource(wsClient, address, "token", false, 100) + new GitlabSource(wsClient, address, "token", false, 100, 1) .getSshKeys(Set(GitlabUser(1, "usr1"))) .value .failed