Update license audit toolset and license list for new requirements.txt.

2 files changed, 86 insertions, 53 deletions

diff --git a/tools/license-audit/TowerLicenses.csv b/tools/license-audit/TowerLicenses.csv
index be8843361c..a7d6ff055e 100644
--- a/tools/license-audit/TowerLicenses.csv
+++ b/tools/license-audit/TowerLicenses.csv
@@ -1,7 +1,8 @@
 Babel,BSD,http://babel.pocoo.org/,pip
-IPy,BSD,https://github.com/autocracy/python-ipy,pip
+Django,BSD,http://www.djangoproject.com/,pip
 Markdown,BSD,http://packages.python.org/Markdown/,pip
 PrettyTable,BSD,http://code.google.com/p/prettytable/,pip
+PyYAML,MIT,http://pyyaml.org/wiki/PyYAML,pip
 South,Apache 2.0,http://south.aeracode.org/,pip
 amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip
 angular,MIT,https://github.com/angular/angular.js.git,js
@@ -27,8 +28,12 @@ bootstrap,MIT,https://github.com/twbs/bootstrap.git,js
 bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js
 boto,MIT,https://github.com/boto/boto/,pip
 celery,BSD,http://celeryproject.org,pip
+cffi,MIT,http://cffi.readthedocs.org,pip
+cliff,Apache 2.0,https://launchpad.net/python-cliff,pip
+cmd2,MIT,http://packages.python.org/cmd2/,pip
 codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js
 components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js
+cryptography,BSD or Apache 2.0,https://github.com/pyca/cryptography,pip
 d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip
 d3,BSD,https://github.com/mbostock/d3.git,js
 distribute,PSF or ZPL,http://packages.python.org/distribute,pip
@@ -37,6 +42,7 @@ django-celery,BSD,http://celeryproject.org,pip
 django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip
 django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip
 django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip
+django-polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip
 django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip
 django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip
 django-split-settings,BSD,http://github.com/2general/django-split-settings,pip
@@ -46,10 +52,16 @@ djangorestframework,BSD,http://www.django-rest-framework.org,pip
 dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip
 dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip
 ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js
+enum34,BSD,https://pypi.python.org/pypi/enum34,pip
+gevent,MIT,http://www.gevent.org/,pip
 gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip
 gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip
+greenlet,MIT,https://github.com/python-greenlet/greenlet,pip
 httplib2,MIT,https://github.com/jcgregorio/httplib2,pip
+idna,BSD,https://github.com/kjd/idna,pip
 importlib,PSF,https://pypi.python.org/pypi/importlib,pip
+ipaddress,PSF,https://github.com/phihag/ipaddress,pip
+ipy,BSD,https://github.com/autocracy/python-ipy,pip
 iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip
 isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip
 jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js
@@ -59,19 +71,28 @@ jquery-ui,MIT,http://jqueryui.com/,js
 jqueryui,MIT,http://jqueryui.com/,js
 js-yaml,MIT,https://github.com/nodeca/js-yaml,js
 jsonlint,MIT,https://github.com/zaach/jsonlint.git,js
+jsonpatch,BSD,https://github.com/stefankoegl/python-json-patch,pip
+jsonpointer,BSD,https://github.com/stefankoegl/python-json-pointer,pip
+jsonschema,MIT,http://github.com/Julian/jsonschema,pip
 kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js
 keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip
 kombu,BSD,http://kombu.readthedocs.org,pip
 loader.js,MIT,https://github.com/stefanpenner/loader.js,js
 lodash,MIT,https://github.com/lodash/lodash,js
 lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js
+lxml,BSD,http://lxml.de/,pip
 mock,BSD,http://www.voidspace.org.uk/python/mock/,pip
 moment,MIT,http://momentjs.com/,js
 mongoengine,MIT,http://mongoengine.org/,pip
+msgpack-python,Apache 2.0,http://msgpack.org/,pip
 netaddr,BSD,https://github.com/drkjam/netaddr/,pip
+netifaces,MIT,https://bitbucket.org/al45tair/netifaces,pip
 nvd3,Apache 2.0,http://www.nvd3.org,js
 ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip
 os-client-config,Apache 2.0,http://www.openstack.org/,pip
+os-diskconfig-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
+os-networksv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
+os-virtual-interfacesv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
 os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
 os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
 os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
@@ -82,21 +103,33 @@ oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip
 pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip
 pexpect,ISC,http://pexpect.readthedocs.org/,pip
 pip,MIT,http://www.pip-installer.org,pip
+prettytable,BSD,http://code.google.com/p/prettytable/,pip
 psphere,Apache 2.0,https://github.com/jkinred/psphere,pip
+psycopg2,LGPL with exceptions or ZPL,http://initd.org/psycopg/,pip
+pyOpenSSL,Apache 2.0,https://github.com/pyca/pyopenssl,pip
+pyasn1,BSD,http://sourceforge.net/projects/pyasn1/,pip
+pycparser,BSD,https://github.com/eliben/pycparser,pip
+pycrypto,Public domain,http://www.pycrypto.org/,pip
+pymongo,Apache 2.0,http://github.com/mongodb/mongo-python-driver,pip
+pyparsing,MIT,http://pyparsing.wikispaces.com/,pip
 pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip
 python-cinderclient,Apache 2.0,http://www.openstack.org/,pip
 python-dateutil,BSD,https://dateutil.readthedocs.org,pip
 python-glanceclient,Apache 2.0,http://www.openstack.org/,pip
 python-ironicclient,Apache 2.0,http://www.openstack.org/,pip
 python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip
+python-ldap,Python style,http://www.python-ldap.org/,pip
 python-neutronclient,Apache 2.0,http://www.openstack.org/,pip
 python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip
 python-swiftclient,Apache 2.0,http://www.openstack.org/,pip
 python-troveclient,Apache 2.0,http://www.openstack.org/,pip
 pytz,MIT,http://pythonhosted.org/pytz,pip
 pywinrm,MIT,http://github.com/diyan/pywinrm/,pip
+pyzmq,LGPL+BSD,http://github.com/zeromq/pyzmq,pip
 rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip
 rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip
+rax-default-network-flags-python-novaclient-ext,Apache 2.0,https://github.com/rackspace/rax_default_network_flags_python_novaclient_ext,pip
+rax-scheduled-images-python-novaclient-ext,Apache 2.0,https://github.com/rackspace-titan/rax_scheduled_images_python_novaclient_ext,pip
 rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackspace/rax_default_network_flags_python_novaclient_ext,pip
 rax_scheduled_images_python_novaclient_ext,Apache 2.0,https://github.com/rackspace-titan/rax_scheduled_images_python_novaclient_ext,pip
 redis,MIT,http://github.com/andymccurdy/redis-py,pip
@@ -112,7 +145,11 @@ sizzle,MIT,https://github.com/jquery/sizzle.git,js
 socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js
 stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip
 suds,LGPL 3,https://fedorahosted.org/suds,pip
+superlance,BSD,http://supervisord.org,pip
 timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js
 twitter,MIT,http://getbootstrap.com,js
 underscore,MIT,https://github.com/jashkenas/underscore,js
+warlock,Apache 2.0,http://github.com/bcwaldon/warlock,pip
+wheel,MIT,http://bitbucket.org/pypa/wheel/,pip
+wsgiref,PSF or ZPL,http://cheeseshop.python.org/pypi/wsgiref,pip
 xmltodict,MIT,https://github.com/martinblech/xmltodict,pip
diff --git a/tools/license-audit/license-audit.py b/tools/license-audit/license-audit.py
index c33092a6d7..e5a71ec8e2 100755
--- a/tools/license-audit/license-audit.py
+++ b/tools/license-audit/license-audit.py
@@ -17,52 +17,55 @@ def usage():
     sys.exit(1)
 
 def read_requirements(towerpath):
-    filename = '%s/awx/lib/site-packages/README' % (towerpath,)
+    filename = '%s/requirements/requirements.txt' % (towerpath,)
     ret = {}
-    f = open(filename)
-    if not f:
+    try:
+        f = open(filename)
+    except:
         print "failed to open %s" %(filename,)
         return None
     data = f.readlines()
     f.close()
     for line in data:
+        if line[0] == '#':
+            continue
         if '==' in line:
-            m = re.match(r"(\S+)==(\S+) \((\S+)",line)
+            m = re.match(r"(\S+)==(\S+)",line)
             if m:
                 name = m.group(1)
                 version = m.group(2)
-                pathname = m.group(3)
-                if pathname.endswith(',') or pathname.endswith(')'):
-                    pathname = pathname[:-1]
-                if pathname.endswith('/*'):
-                    pathname = pathname[:-2]
                 item = {}
                 item['name'] = name
                 item['version'] = version
-                item['path'] = pathname
                 ret[name] = item
-    return ret
-
-def get_python(towerpath):
-    excludes = [ 
-        'README*', 
-        '*.dist-info', 
-        'funtests', 
-        'easy_install.py', 
-        'oslo', 
-        'pkg_resources', 
-        '_markerlib'
-    ]
-    directory = '%s/awx/lib/site-packages' % (towerpath,)
-    dirlist = os.listdir(directory)
-    ret = []
-    for item in dirlist:
-        use = True
-        for exclude in excludes:
-            if fnmatch.fnmatch(item, exclude):
-                use = False
-        if use:
-            ret.append(item)
+                continue
+        elif line.startswith("git+https"):
+            l = line.rsplit('/',1)
+            m = re.match(r"(\S+).git@(\S+)#",l[1])
+            if m:
+                name = m.group(1)
+                version = m.group(2)
+                if version.startswith('tower_'):
+                    version = version[6:]
+                item = {}
+                if name == 'python-ipy':
+                    name='ipy'
+                item['name'] = name
+                item['version'] = version
+                if len(version) > 20:
+                    # it's a sha1sum, read it off the egg spec
+                    lver = l[1].rsplit('-',1)
+                    if lver[1] == l[1]:
+                        lver = l[1].rsplit('_',1)
+                    item['version'] = lver[1][:-1]
+                ret[name] = item
+                continue
+        else:
+            item = {}
+            item['name'] = line[:-1]
+            item['version'] = ''
+            ret[name] = item
+            continue
     return ret
 
 def get_js(towerpath):
@@ -116,13 +119,13 @@ def normalize_license(license):
     license = license.replace('"','')
     if license == 'None':
         return 'UNKNOWN'
-    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']:
+    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0', 'APL2']:
         return 'Apache 2.0'
     if license == 'ISC license':
         return 'ISC'
     if license == 'MIT License' or license == 'MIT license':
         return 'MIT'
-    if license == 'BSD License' or license == 'Simplified BSD':
+    if license in ['BSD License', 'Simplified BSD', 'BSD-derived (http://www.repoze.org/LICENSE.txt)', 'BSD-like', 'Modified BSD License']:
         return 'BSD'
     if license == 'LGPL':
         return 'LGPL 2.1'
@@ -131,6 +134,10 @@ def normalize_license(license):
         return 'Apache 2.0'
     if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1:
         return 'MIT'
+    if license == '"BSD or Apache License, Version 2.0"':
+        return 'BSD or Apache 2.0'
+    if license == 'Modified BSD License':
+        return 'BSD'
     if license == 'Python Software Foundation License':
         return 'PSF'
     return license
@@ -183,24 +190,11 @@ if not olddata or not requirements:
     print "No starting data"
     sys.exit(1)
 
-# Get directory of vendored things from site-packages...
-python_packages = get_python(tower_path)
-
-# ... and ensure they're noted in the requirements file
-ok = True
-for package in python_packages:
-    if not search_requirements(requirements, package):
-        print "%s not in requirements!" % (package,)
-        ok = False
-if not ok:
-    sys.exit(1)
-
-
 # See if there's pip things in our current license list that we don't have now
 reqs = requirements.keys()
 for item in olddata.values():
     if item['source'] == 'pip' and item['name'] not in reqs:
-        print "No longer vendoring %s" %(item['name'],)
+        print "Potentially no longer vendoring %s" %(item['name'],)
 
 # Get directory of vendored JS things from the js dir
 js_packages = get_js(tower_path)
@@ -214,10 +208,12 @@ for item in olddata.values():
 # Take the requirements file, and get license information where necessary
 cs = yolk.pypi.CheeseShop()
 for req in requirements.values():
-    cs_info = cs.release_data(req['name'],req['version'])
+    # name sanitization
+    (pname, pvers) = cs.query_versions_pypi(req['name'])
+    cs_info = cs.release_data(pname,req['version'])
     if not cs_info:
-        print "Couldn't find '%s-%s'" %(req['name'],req['version'])
-        if 'name' not in olddata:
+        print "Couldn't find '%s==%s'" %(req['name'],req['version'])
+        if req['name'] not in olddata:
             print "... and it's not in the current data. This needs fixed!"
             sys.exit(1)
         continue
@@ -241,7 +237,7 @@ for req in requirements.values():
 
 # Update JS package info
 for pkg in js:
-    if 'pkg' in olddata:
+    if pkg in olddata:
         data = olddata[pkg]
         new = js_packages[pkg]
         if new['license'] != 'UNKNOWN' and new['license'] != data['license']: