Adding awx_ as well as tower_ variable names for webhooks (#11925)

Adding utility to ease testing webhooks from command line
Modifying all variables to use a constants list of variable names

3 files changed, 363 insertions, 0 deletions

diff --git a/tools/scripts/post_webhook.py b/tools/scripts/post_webhook.py
new file mode 100755
index 0000000000..766a033bd1
--- /dev/null
+++ b/tools/scripts/post_webhook.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+from hashlib import sha1
+from sys import exit
+import click
+import hmac
+import http.client as http_client
+import json
+import logging
+import requests
+import urllib3
+import uuid
+
+
+@click.command()
+@click.option('--file', required=True, help='File containing the post data.')
+@click.option('--key', "webhook_key", required=True, help='The webhook key for the job template.')
+@click.option('--url', required=True, help='The webhook url for the job template (i.e. https://tower.jowestco.net:8043/api/v2/job_templates/637/github/.')
+@click.option('--event-type', help='Specific value for Event header, defaults to "issues" for GitHub and "Push Hook" for GitLab')
+@click.option('--verbose', is_flag=True, help='Dump HTTP communication for debugging')
+@click.option('--insecure', is_flag=True, help='Ignore SSL certs if true')
+def post_webhook(file, webhook_key, url, verbose, event_type, insecure):
+    """
+    Helper command for submitting POST requests to Webhook endpoints.
+
+    We have two sample webhooks in tools/scripts/webhook_examples for gitlab and github.
+    These or any other file can be pointed to with the --file parameter.
+
+    \b
+    Additional example webhook events can be found online.
+        For GitLab see:
+        https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html
+
+    \b
+        For GitHub see:
+        https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads
+
+    \b
+    For setting up webhooks in AWX see:
+        https://docs.ansible.com/ansible-tower/latest/html/userguide/webhooks.html
+
+    \b
+    Example usage for GitHub:
+      ./post_webhook.py \\
+        --file webhook_examples/github_push.json \\
+        --url https://tower.jowestco.net:8043/api/v2/job_templates/637/github/ \\
+        --key AvqBR19JDFaLTsbF3p7FmiU9WpuHsJKdHDfTqKXyzv1HtwDGZ8 \\
+        --insecure \\
+        --type github
+    
+    \b
+    Example usage for GitLab:
+      ./post_webhook.py \\
+        --file webhook_examples/gitlab_push.json \\
+        --url https://tower.jowestco.net:8043/api/v2/job_templates/638/gitlab/ \\
+        --key fZ8vUpfHfb1Dn7zHtyaAsyZC5IHFcZf2a2xiBc2jmrBDptCOL2 \\
+        --insecure \\
+        --type=gitlab 
+
+    \b
+    NOTE: GitLab webhooks are stored in the DB with a UID of the hash of the POST body.
+          After submitting one post GitLab post body a second POST of the same payload 
+          can result in a response like: 
+              Response code: 202
+              Response body:
+              {
+                  "message": "Webhook previously received, aborting."
+              }
+
+          If you need to test multiple GitLab posts simply change your payload slightly
+
+    """
+    if insecure:
+        # Disable insecure warnings
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    if verbose:
+        # Enable HTTP debugging
+        http_client.HTTPConnection.debuglevel = 1
+        # Configure logging
+        logging.basicConfig()
+        logging.getLogger().setLevel(logging.DEBUG)
+        requests_log = logging.getLogger("requests.packages.urllib3")
+        requests_log.setLevel(logging.DEBUG)
+        requests_log.propagate = True
+
+    # read webhook payload
+    with open(file, 'r') as f:
+        post_data = json.loads(f.read())
+
+    # Construct Headers
+    headers = {
+        'Content-Type': 'application/json',
+    }
+
+    # Encode key and post_data
+    key_bytes = webhook_key.encode('utf-8', 'strict')
+    data_bytes = str(json.dumps(post_data)).encode('utf-8', 'strict')
+
+    # Compute sha1 mac
+    mac = hmac.new(key_bytes, msg=data_bytes, digestmod=sha1)
+
+    if url.endswith('/github/'):
+        headers.update(
+            {
+                'X-Hub-Signature': 'sha1={}'.format(mac.hexdigest()),
+                'X-GitHub-Event': 'issues' if event_type == 'default' else event_type,
+                'X-GitHub-Delivery': str(uuid.uuid4()),
+            }
+        )
+    elif url.endswith('/gitlab/'):
+        mac = hmac.new(key_bytes, msg=data_bytes, digestmod=sha1)
+        headers.update(
+            {
+                'X-GitLab-Event': 'Push Hook' if event_type == 'default' else event_type,
+                'X-GitLab-Token': webhook_key,
+            }
+        )
+    else:
+        click.echo("This utility only knows how to support URLs that end in /github/ or /gitlab/.")
+        exit(250)
+
+    # Make post
+    r = requests.post(url, data=json.dumps(post_data), headers=headers, verify=(not insecure))
+
+    if not verbose:
+        click.echo("Response code: {}".format(r.status_code))
+        click.echo("Response body:")
+    try:
+        click.echo(json.dumps(r.json(), indent=4))
+    except:
+        click.echo(r.text)
+
+
+if __name__ == '__main__':
+    post_webhook()
diff --git a/tools/scripts/webhook_examples/github_push.json b/tools/scripts/webhook_examples/github_push.json
new file mode 100644
index 0000000000..4805c2881f
--- /dev/null
+++ b/tools/scripts/webhook_examples/github_push.json
@@ -0,0 +1,156 @@
+{
+  "ref": "refs/tags/simple-tag",
+  "before": "0000000000000000000000000000000000000000",
+  "after": "6113728f27ae82c7b1a177c8d03f9e96e0adf246",
+  "created": true,
+  "deleted": false,
+  "forced": false,
+  "base_ref": "refs/heads/main",
+  "compare": "https://github.com/Codertocat/Hello-World/compare/simple-tag",
+  "commits": [],
+  "head_commit": {
+    "id": "6113728f27ae82c7b1a177c8d03f9e96e0adf246",
+    "tree_id": "4b825dc642cb6eb9a060e54bf8d69288fbee4904",
+    "distinct": true,
+    "message": "Adding a .gitignore file",
+    "timestamp": "2019-05-15T15:20:41Z",
+    "url": "https://github.com/Codertocat/Hello-World/commit/6113728f27ae82c7b1a177c8d03f9e96e0adf246",
+    "author": {
+      "name": "Codertocat",
+      "email": "21031067+Codertocat@users.noreply.github.com",
+      "username": "Codertocat"
+    },
+    "committer": {
+      "name": "Codertocat",
+      "email": "21031067+Codertocat@users.noreply.github.com",
+      "username": "Codertocat"
+    },
+    "added": [
+      ".gitignore"
+    ],
+    "removed": [],
+    "modified": []
+  },
+  "repository": {
+    "id": 186853002,
+    "node_id": "MDEwOlJlcG9zaXRvcnkxODY4NTMwMDI=",
+    "name": "Hello-World",
+    "full_name": "Codertocat/Hello-World",
+    "private": false,
+    "owner": {
+      "name": "Codertocat",
+      "email": "21031067+Codertocat@users.noreply.github.com",
+      "login": "Codertocat",
+      "id": 21031067,
+      "node_id": "MDQ6VXNlcjIxMDMxMDY3",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/Codertocat",
+      "html_url": "https://github.com/Codertocat",
+      "followers_url": "https://api.github.com/users/Codertocat/followers",
+      "following_url": "https://api.github.com/users/Codertocat/following{/other_user}",
+      "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions",
+      "organizations_url": "https://api.github.com/users/Codertocat/orgs",
+      "repos_url": "https://api.github.com/users/Codertocat/repos",
+      "events_url": "https://api.github.com/users/Codertocat/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/Codertocat/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/Codertocat/Hello-World",
+    "description": null,
+    "fork": false,
+    "url": "https://github.com/Codertocat/Hello-World",
+    "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks",
+    "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams",
+    "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks",
+    "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events",
+    "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags",
+    "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages",
+    "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers",
+    "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors",
+    "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers",
+    "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription",
+    "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges",
+    "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads",
+    "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}",
+    "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments",
+    "created_at": 1557933565,
+    "updated_at": "2019-05-15T15:20:41Z",
+    "pushed_at": 1557933657,
+    "git_url": "git://github.com/Codertocat/Hello-World.git",
+    "ssh_url": "git@github.com:Codertocat/Hello-World.git",
+    "clone_url": "https://github.com/Codertocat/Hello-World.git",
+    "svn_url": "https://github.com/Codertocat/Hello-World",
+    "homepage": null,
+    "size": 0,
+    "stargazers_count": 0,
+    "watchers_count": 0,
+    "language": "Ruby",
+    "has_issues": true,
+    "has_projects": true,
+    "has_downloads": true,
+    "has_wiki": true,
+    "has_pages": true,
+    "forks_count": 1,
+    "mirror_url": null,
+    "archived": false,
+    "disabled": false,
+    "open_issues_count": 2,
+    "license": null,
+    "forks": 1,
+    "open_issues": 2,
+    "watchers": 0,
+    "default_branch": "master",
+    "stargazers": 0,
+    "master_branch": "master"
+  },
+  "pusher": {
+    "name": "Codertocat",
+    "email": "21031067+Codertocat@users.noreply.github.com"
+  },
+  "sender": {
+    "login": "Codertocat",
+    "id": 21031067,
+    "node_id": "MDQ6VXNlcjIxMDMxMDY3",
+    "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/Codertocat",
+    "html_url": "https://github.com/Codertocat",
+    "followers_url": "https://api.github.com/users/Codertocat/followers",
+    "following_url": "https://api.github.com/users/Codertocat/following{/other_user}",
+    "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions",
+    "organizations_url": "https://api.github.com/users/Codertocat/orgs",
+    "repos_url": "https://api.github.com/users/Codertocat/repos",
+    "events_url": "https://api.github.com/users/Codertocat/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/Codertocat/received_events",
+    "type": "User",
+    "site_admin": false
+  }
+}
diff --git a/tools/scripts/webhook_examples/gitlab_push.json b/tools/scripts/webhook_examples/gitlab_push.json
new file mode 100644
index 0000000000..4389e11d1e
--- /dev/null
+++ b/tools/scripts/webhook_examples/gitlab_push.json
@@ -0,0 +1,71 @@
+{
+  "object_kind": "push",
+  "event_name": "push",
+  "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+  "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "ref": "refs/heads/master",
+  "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+  "user_id": 4,
+  "user_name": "John Smith",
+  "user_username": "jsmith",
+  "user_email": "john@example.com",
+  "user_avatar": "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80",
+  "project_id": 15,
+  "project":{
+    "id": 15,
+    "name":"Diaspora",
+    "description":"",
+    "web_url":"http://example.com/mike/diaspora",
+    "avatar_url":null,
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "namespace":"Mike",
+    "visibility_level":0,
+    "path_with_namespace":"mike/diaspora",
+    "default_branch":"master",
+    "homepage":"http://example.com/mike/diaspora",
+    "url":"git@example.com:mike/diaspora.git",
+    "ssh_url":"git@example.com:mike/diaspora.git",
+    "http_url":"http://example.com/mike/diaspora.git"
+  },
+  "repository":{
+    "name": "Diaspora",
+    "url": "git@example.com:mike/diaspora.git",
+    "description": "",
+    "homepage": "http://example.com/mike/diaspora",
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "visibility_level":0
+  },
+  "commits": [
+    {
+      "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+      "message": "Update Catalan translation to e38cb41.\n\nSee https://gitlab.com/gitlab-org/gitlab for more information",
+      "title": "Update Catalan translation to e38cb41.",
+      "timestamp": "2011-12-12T14:27:31+02:00",
+      "url": "http://example.com/mike/diaspora/commit/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+      "author": {
+        "name": "Jordi Mallach",
+        "email": "jordi@softcatala.org"
+      },
+      "added": ["CHANGELOG"],
+      "modified": ["app/controller/application.rb"],
+      "removed": []
+    },
+    {
+      "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "message": "fixed readme",
+      "title": "fixed readme",
+      "timestamp": "2012-01-03T23:36:29+02:00",
+      "url": "http://example.com/mike/diaspora/commit/da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "author": {
+        "name": "GitLab dev user",
+        "email": "gitlabdev@dv6700.(none)"
+      },
+      "added": ["CHANGELOG"],
+      "modified": ["app/controller/application.rb"],
+      "removed": []
+    }
+  ],
+  "total_commits_count": 4
+}