Convert http(s) github urls to github ssh urls

To support private repositories cloning with public-private-key authentication, we're now converting the Github http(s) urls from the payload to github ssh urls.

Author: pvdissel

subprojects/listener/README.md

@@ -3,21 +3,15 @@ Pipeline Listener
 
 Receives and queues service hook notifications, like `push` events from GitHub.
 
-This listener supports multiple providers, GitHub being the first.
+This listener supports the following provider:
+
+- github.com
 
 Usage
 -----
 
 Pipeline Listener (`pipe-listener`) provides a HTTP REST API that providers like GitHub can notify. The API is non-blocking, after basic verification of the notification message, the API will return a success (HTTP 204) or failure (HTTP 422) status. The notification message will be queued for processing a.s.a.p. by its processor.
 
-### GET /
-
-Shows a simple overview page with:
-
-- Loaded providers and their details
-- Queued notifications (with basic details)
-- Amount processed notifications since start-up
-
 ### POST /providers/github
 
 GitHub provider, supports the GitHub WebHook format as explained on [Post-Receive Hooks](https://help.github.com/articles/post-receive-hooks)

subprojects/listener/listener.gradle

@@ -11,11 +11,8 @@ dependencies {
     compile libraries.groovyJson
     testCompile libraries.spock
     testCompile libraries.dropwizardTesting
-
-    // this dropwizard dependency is needed due to http://issues.gradle.org/browse/GRADLE-2647
-//    compile libraries.'jersey-client'
 }
 
-//run {
-//    args 'server', './src/dist/config/helloworld.yml'
-//}
+run {
+    args 'server'
+}

subprojects/listener/src/main/groovy/org/pipelinelabs/pipeline/listen/core/github/GithubUtils.groovy

@@ -0,0 +1,20 @@
+package org.pipelinelabs.pipeline.listen.core.github
+
+
+class GithubUtils {
+
+    private static final String SSH_URL_FORMAT = '[email protected]:%s.git'
+
+    static String toGithubSshUrl(String uri) throws URISyntaxException {
+        return toGithubSshUrl(new URI(uri))
+    }
+
+    static String toGithubSshUrl(URI uri) {
+        return String.format(SSH_URL_FORMAT, cleanUriPath(uri))
+    }
+
+    private static String cleanUriPath(URI uri) {
+        def path = uri.path
+        return path.replaceAll(/\/+/, '/')
+    }
+}

subprojects/listener/src/main/groovy/org/pipelinelabs/pipeline/listen/resources/GitHubWebHookResource.groovy

@@ -11,6 +11,7 @@ import javax.ws.rs.core.Response
 import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST
+import static org.pipelinelabs.pipeline.listen.core.github.GithubUtils.toGithubSshUrl
 
 @Slf4j
 @Path("/providers/github")
@@ -26,19 +27,19 @@ class GitHubWebHookResource {
 
     @POST
     Response trigger(@FormParam('payload') String payload) {
-        final slurper = new JsonSlurper()
+        def slurper = new JsonSlurper()
         def info = slurper.parseText(payload)
         try {
             verifyRequest(info)
         } catch (AssertionError e) {
-            log.warn("Received GitHub WebHook payload with unexpected format:\n{}", info)
+            log.warn("Received GitHub WebHook payload with unexpected format:\n{}", e.message)
             return Response.status(BAD_REQUEST.statusCode).build()
         }
         return handleRequest(info)
     }
 
     private Response handleRequest(request) {
-        def event = new GitTriggerEvent(request.repository.url)
+        def event = new GitTriggerEvent(toGithubSshUrl(request.repository.url))
         bus.post(event)
         Response.noContent().build()
     }
@@ -57,5 +58,14 @@ class GitHubWebHookResource {
         assert request.repository.private == true ||
                 request.repository.private == false
         assert request.repository.url
+        assertValidUri(request.repository.url)
+    }
+
+    private assertValidUri(String uri) throws AssertionError {
+        try {
+            new URI(uri)
+        } catch (URISyntaxException e) {
+            throw new AssertionError(e.message, e)
+        }
     }
 }

subprojects/listener/src/test/groovy/org/pipelinelabs/pipeline/listen/core/DeadEventHandlerSpec.groovy

@@ -9,11 +9,9 @@ import spock.lang.Specification
 
 class DeadEventHandlerSpec extends Specification {
     private EventBus bus = Mock(EventBus)
+    private DeadEventHandler worker = new DeadEventHandler(bus)
 
     def 'Start registers itself to the event bus'() {
-        given:
-        def worker = new DeadEventHandler(bus)
-
         when:
         worker.start()
 
@@ -22,17 +20,14 @@ class DeadEventHandlerSpec extends Specification {
     }
 
     def 'Stop unregisters itself from the event bus'() {
-        given:
-        def worker = new DeadEventHandler(bus)
-
         when:
         worker.stop()
 
         then:
         1 * bus.unregister(worker)
     }
 
-    def 'Work handles DeadEvent and is annotated with @Subscribe'() {
+    def 'work() is the event handler of DeadEvent objects'() {
         def method = DeadEventHandler.getMethod("work", DeadEvent)
 
         expect:

subprojects/listener/src/test/groovy/org/pipelinelabs/pipeline/listen/core/GitWorkerSpec.groovy

@@ -7,11 +7,9 @@ import spock.lang.Specification
 
 class GitWorkerSpec extends Specification {
     private EventBus bus = Mock(EventBus)
+    private GitWorker worker = new GitWorker(bus)
 
     def 'Start registers itself to the event bus'() {
-        given:
-        def worker = new GitWorker(bus)
-
         when:
         worker.start()
 
@@ -20,26 +18,22 @@ class GitWorkerSpec extends Specification {
     }
 
     def 'Stop unregisters itself from the event bus'() {
-        given:
-        def worker = new GitWorker(bus)
-
         when:
         worker.stop()
 
         then:
         1 * bus.unregister(worker)
     }
 
-    def 'Work handles GitTriggerEvent and is annotated with @Subscribe'() {
+    def 'work() is the event handler of GitTriggerEvent objects'() {
         def method = GitWorker.getMethod("work", GitTriggerEvent)
 
         expect:
         method.isAnnotationPresent(Subscribe)
     }
 
-    @Ignore('Needs to be testable somehow')
+    @Ignore
     def 'Work clones the git repo and starts pipe-runner'() {
-        def worker = new GitWorker(bus)
         final event = Mock(GitTriggerEvent)
 
         expect:

subprojects/listener/src/test/groovy/org/pipelinelabs/pipeline/listen/core/github/GithubUtilsSpec.groovy

@@ -0,0 +1,36 @@
+package org.pipelinelabs.pipeline.listen.core.github
+
+import spock.lang.Specification
+import spock.lang.Unroll
+
+
+class GithubUtilsSpec extends Specification {
+
+    @Unroll
+    def 'Convert new URI("#uri") to github ssh url "#expected"'() {
+        expect:
+        final actual = GithubUtils.toGithubSshUrl(new URI(uri))
+        actual == expected
+
+        where:
+        uri                                    || expected
+        'https://github.com/octokitty/testing' || '[email protected]:/octokitty/testing.git'
+        'http://github.com/octokitty/testing'  || '[email protected]:/octokitty/testing.git'
+        'http://github.com//octokitty/testing' || '[email protected]:/octokitty/testing.git'
+        'http://github.com/octokitty//testing' || '[email protected]:/octokitty/testing.git'
+    }
+
+    @Unroll
+    def 'Convert "#uri" to github ssh url "#expected"'() {
+        expect:
+        final actual = GithubUtils.toGithubSshUrl(uri)
+        actual == expected
+
+        where:
+        uri                                    || expected
+        'https://github.com/octokitty/testing' || '[email protected]:/octokitty/testing.git'
+        'http://github.com/octokitty/testing'  || '[email protected]:/octokitty/testing.git'
+        'http://github.com//octokitty/testing' || '[email protected]:/octokitty/testing.git'
+        'http://github.com/octokitty//testing' || '[email protected]:/octokitty/testing.git'
+    }
+}

subprojects/listener/src/test/groovy/org/pipelinelabs/pipeline/listen/resources/GitHubWebHookResourceSpec.groovy

@@ -7,6 +7,7 @@ import com.sun.jersey.api.client.WebResource
 import com.sun.jersey.api.representation.Form
 import com.yammer.dropwizard.testing.ResourceTest
 import org.pipelinelabs.pipeline.listen.core.GitTriggerEvent
+import spock.lang.Ignore
 import spock.lang.Specification
 import spock.lang.Unroll
 
@@ -21,7 +22,28 @@ class GitHubWebHookResourceSpec extends Specification {
     def eventBus = Mock(EventBus)
     TestResource resource = new TestResource(eventBus)
 
-    def 'Correct payload results in event on queue'() {
+    @Ignore
+    def 'Valid payload results in git event with github ssh url on queue'() {
+        given:
+        def form = new Form()
+        form.add('payload', jsonFixture("github/samplePayload.json"))
+
+        when:
+        def response = requestBuilder()
+                .type(APPLICATION_FORM_URLENCODED_TYPE)
+                .accept(APPLICATION_JSON_TYPE)
+                .post(ClientResponse, form)
+
+        then:
+        1 * eventBus.post(_ as GitTriggerEvent) >> { event ->
+            assert event.url == '[email protected]:/octokitty/testing.git'
+        }
+        response.type == APPLICATION_JSON_TYPE
+        response.clientResponseStatus == NO_CONTENT
+        response.length == -1
+    }
+
+    def 'Valid payload results in git event on queue'() {
         given:
         def form = new Form()
         form.add('payload', jsonFixture("github/samplePayload.json"))

subprojects/runner/runner.gradle

@@ -3,7 +3,7 @@ apply plugin: 'application'
 apply from: script('cucumber')
 
 applicationName = 'pipe-runner'
-description = 'pipe-runner --the pipeline runner, executes the project pipeline from the project.pipe configuration'
+description = 'pipe-runner --the pipeline runner, executes the project pipeline a pipeline configuration file'
 mainClassName = 'org.pipelinelabs.pipeline.runner.cli.Pipeline'
 
 dependencies {

subprojects/runner/src/main/groovy/org/pipelinelabs/pipeline/runner/cli/command/RunCommand.java

@@ -15,13 +15,13 @@
 
 @Parameters(
         commandNames = {RunCommand.NAME}
-        , commandDescription = "Run, baby run!"
+        , commandDescription = "Run pipeline"
 )
 public class RunCommand implements Command {
     public static final String NAME = "run";
 
     @Parameter(
-            description = ""
+            description = "Pipeline configuration to run"
             , listConverter = FileConverter.class
             , required = true
             , arity = 1