aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Fincham <michael@hotplate.co.nz>2017-11-19 21:52:32 +1300
committerMichael Fincham <michael@hotplate.co.nz>2017-11-19 21:52:32 +1300
commit48010e45a6ede46d5a9fb4eb079e82c239236b78 (patch)
tree8cce38ee5fd88826df0685f2088ba4b5c26f5a8c
parent634a5a23ee7b9fc3a39b19342946416561dc5dce (diff)
downloadadvisory-feeds-48010e45a6ede46d5a9fb4eb079e82c239236b78.tar.gz
advisory-feeds-48010e45a6ede46d5a9fb4eb079e82c239236b78.tar.bz2
advisory-feeds-48010e45a6ede46d5a9fb4eb079e82c239236b78.zip
Initial upload of demo code for bsides
-rw-r--r--advisories/__init__.py0
-rw-r--r--advisories/admin.py61
-rw-r--r--advisories/management/__init__.py0
-rw-r--r--advisories/management/commands/__init__.py0
-rw-r--r--advisories/management/commands/updateadvisories.py445
-rw-r--r--advisories/models.py143
-rw-r--r--advisorybrowser/__init__.py0
-rw-r--r--advisorybrowser/settings.py168
-rw-r--r--advisorybrowser/urls.py7
-rw-r--r--advisorybrowser/wsgi.py16
-rw-r--r--browser/__init__.py0
-rw-r--r--browser/admin.py3
-rw-r--r--browser/apps.py5
-rw-r--r--browser/models.py3
-rw-r--r--browser/templates/browser/advisory.html116
-rw-r--r--browser/templates/browser/base.html76
-rw-r--r--browser/templates/browser/cves.html49
-rw-r--r--browser/templates/browser/index.html82
-rw-r--r--browser/templates/browser/vulnerability.html37
-rw-r--r--browser/templatetags/__init__.py0
-rw-r--r--browser/templatetags/advisory_fields.py45
-rw-r--r--browser/urls.py12
-rw-r--r--browser/views.py108
-rwxr-xr-xmanage.py22
-rw-r--r--requirements.txt15
25 files changed, 1413 insertions, 0 deletions
diff --git a/advisories/__init__.py b/advisories/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/advisories/__init__.py
diff --git a/advisories/admin.py b/advisories/admin.py
new file mode 100644
index 0000000..1eb9a59
--- /dev/null
+++ b/advisories/admin.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2017 Catalyst.net Ltd
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: Michael Fincham <michael.fincham@catalyst.net.nz>
+# Author: Filip Vujicic <filip.vujicic@catalyst.net.nz>
+# Author: Sam Banks <sam.banks@catalyst.net.nz>
+
+from django.contrib import admin
+from django.db.models import Count
+
+from .models import *
+
+class BinaryPackageInline(admin.TabularInline):
+ model = BinaryPackage
+ extra = 0
+ readonly_fields = ('source_package',)
+ fields = ('package', 'safe_version', 'architecture')
+
+class SourcePackageInline(admin.TabularInline):
+ model = SourcePackage
+ extra = 0
+ fields = ('package', 'release', 'safe_version')
+
+class VulnerabilityInline(admin.TabularInline):
+ model = Advisory.vulnerabilities.through
+ extra = 0
+
+class AdvisoryAdmin(admin.ModelAdmin):
+ inlines = [VulnerabilityInline, SourcePackageInline, BinaryPackageInline]
+ list_filter = ['issued', 'source']
+ search_fields = ['upstream_id']
+ list_display = ['upstream_id', 'short_description', 'source_package_names', 'source', 'issued']
+ ordering = ['-issued']
+
+class VulnerabilityAdmin(admin.ModelAdmin):
+ list_filter = ['first_seen']
+ search_fields = ['upstream_id']
+ list_display = ['upstream_id', 'advisory_count', 'first_seen']
+ ordering = ['-first_seen']
+
+ def get_queryset(self, request):
+ qs = super(VulnerabilityAdmin, self).get_queryset(request)
+ return qs.annotate(advisory_count=Count('advisories'))
+
+ def advisory_count(self, inst):
+ return inst.advisory_count
+ advisory_count.admin_order_field = 'advisory_count'
+
+admin.site.register(Advisory, AdvisoryAdmin)
+admin.site.register(Vulnerability, VulnerabilityAdmin)
diff --git a/advisories/management/__init__.py b/advisories/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/advisories/management/__init__.py
diff --git a/advisories/management/commands/__init__.py b/advisories/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/advisories/management/commands/__init__.py
diff --git a/advisories/management/commands/updateadvisories.py b/advisories/management/commands/updateadvisories.py
new file mode 100644
index 0000000..1d205c3
--- /dev/null
+++ b/advisories/management/commands/updateadvisories.py
@@ -0,0 +1,445 @@
+# Copyright (c) 2017 Catalyst.net Ltd
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: Michael Fincham <michael.fincham@catalyst.net.nz>
+# Author: Filip Vujicic <filip.vujicic@catalyst.net.nz>
+# Author: Sam Banks <sam.banks@catalyst.net.nz>
+
+import time
+from datetime import datetime
+import bz2
+import lzma
+import json
+import os
+import re
+
+from bs4 import BeautifulSoup
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction, IntegrityError
+
+from dateutil.parser import parse as dateutil_parse
+import deb822
+
+import apt_inst
+import apt
+
+import pytz
+import requests
+import svn.remote
+
+
+from advisories.models import Advisory, SourcePackage, BinaryPackage, Vulnerability
+
+import logging
+
+if settings.DEBUG is True:
+ logging.basicConfig(format='%(asctime)s | %(levelname)s: %(message)s', level=logging.DEBUG)
+else:
+ logging.basicConfig(format='%(asctime)s | %(levelname)s: %(message)s', level=logging.ERROR)
+
+
+class DebianFeed(object):
+ """
+ Syncs additions to the official DSA list in to the local database, as well as retrieving and parsing metadata about each one.
+ """
+
+ def __init__(self, secure_testing_url=None, cache_location=None, releases=None, architectures=None, snapshot_url=None, security_apt_url=None, list_location=None):
+ self.secure_testing_url = secure_testing_url or "svn://anonscm.debian.org/svn/secure-testing"
+ self.client = svn.remote.RemoteClient(self.secure_testing_url)
+ self.cache_location = cache_location or "%s/advisory_cache/dsa" % settings.BASE_DIR
+ self.releases = releases or (
+ 'wheezy',
+ 'jessie',
+ 'stretch',
+ )
+ self.architectures = architectures or (
+ 'i386',
+ 'amd64',
+ 'all',
+ )
+ self.snapshot_url = snapshot_url or "http://snapshot.debian.org"
+ self.security_apt_url = security_apt_url or "http://security.debian.org/debian-security"
+ self.list_location = list_location or "data/DSA/list"
+
+ def _update_svn_repository(self):
+ """
+ Update the local cache of the DSA list.
+ """
+
+ try:
+ os.makedirs(self.cache_location)
+ except OSError: # directory already exists
+ # XXX: permission denied, full, etc.
+ pass
+
+ try:
+ advisory_list = self.client.cat(self.list_location).decode('utf-8')
+ with open('%s/list' % self.cache_location, 'w') as advisory_list_file:
+ advisory_list_file.write(advisory_list)
+ except ValueError:
+ raise Exception("unable to retrieve data from SVN")
+ except:
+ raise Exception("unknown error updating DSA list cache file")
+
+ def _parse_svn_advisories(self):
+ """
+ Parse the local cache of the DSA/DLA list.
+ """
+
+ advisories = {}
+ with open('%s/list' % self.cache_location) as advisory_list_file:
+ advisory = ''
+ packages = {}
+ cves = []
+ for line in advisory_list_file:
+
+ # minimal state machine follows
+ if line.startswith('['): # start of the DSA/DLA
+ if advisory != '' and len(packages) > 0: # at least one complete DSA/DLA parsed
+ advisories[advisory] = {
+ 'packages': packages,
+ 'description': description,
+ 'issued': issued,
+ 'cves': cves,
+ }
+ issued = pytz.utc.localize(dateutil_parse(line.split('] ')[0].strip('[')))
+ advisory = line.split('] ')[-1].split()[0] # upstream ID of DSA/DLA
+ '''
+ there are at least two advisories, DLA-359-1 and DLA-73-1, which don't
+ follow the "[<date>] <DLA #> <source package> - <description>" format,
+ hence the following code.
+ '''
+ if ' - ' in line:
+ description = line.split(' - ')[-1].strip()
+ else:
+ description = line.split(' ', 5)[-1].strip()
+
+ packages = {}
+ elif line.startswith('\t['): # source package name for a particular release
+
+ if '<' in line: # package has some tags
+ tags = [tag.strip('<>') for tag in line.split() if tag.startswith('<') and tag.endswith('>')]
+ else:
+ tags = []
+
+ if 'not-affected' in tags: # ignore package
+ continue
+
+ release = line.split()[0].strip("\t[] ")
+ if release not in self.releases: # no point in looking for unsupported releases
+ continue
+
+ if 'unfixed' in tags or 'end-of-life' in tags:
+ version = '0' # unsafe at any speed
+ else:
+ version = line.split()[3]
+
+ source_package = line.split()[2]
+ if source_package not in packages:
+ packages[source_package] = {}
+ packages[source_package][release] = version
+ elif line.startswith('\t{CVE'): # list of relevant CVEs fixed in this release
+ cves = line.strip().strip('{}').split()
+
+ return advisories
+
+ def update_local_database(self):
+ """
+ Update the local repository, parse it and add any new advisories to the local database.
+ """
+
+ print(" Updating DSA RDF feed... ", end='')
+ try:
+ dsa_rdf_soup = BeautifulSoup(requests.get('https://www.debian.org/security/dsa-long').content, 'html.parser')
+ dsa_descriptions = {i.attrs['rdf:about'].split('/')[-1].lower():BeautifulSoup(i.description.text, 'html.parser').get_text().strip() for i in dsa_rdf_soup.find_all('item')}
+ print("OK")
+ except:
+ print("could not update DSA RDF feed")
+ dsa_descriptions = {}
+
+ print(" Updating security repository data... ", end='')
+
+ release_metadata = {}
+ source_packages = {}
+
+ # grab the release metadata from the repository
+ for release_name in self.releases:
+ release_metadata[release_name] = deb822.Release(requests.get("%s/dists/%s/updates/Release" % (self.security_apt_url, release_name)).text)
+
+
+ # grab the binary package metadata for the desired architectures
+ # this section attempts to make a reverse mapping for working out what binary packages a particular source package builds
+ for release_name, release_metadatum in release_metadata.items():
+
+ # Chooses which filetype to use
+ if 'Packages.xz\n' in str(release_metadatum):
+ package_filetype = 'xz'
+ elif 'Packages.bz2\n' in str(release_metadatum):
+ package_filetype = 'bz2'
+ else:
+ raise Exception("Unknown package type")
+
+ for component in release_metadatum['Components'].split():
+ for architecture in [architecture for architecture in release_metadatum['Architectures'].split() if architecture in self.architectures]:
+
+ # Gets and decompresses the package data
+ packages_url = "%s/dists/%s/%s/binary-%s/Packages.%s" % (self.security_apt_url, release_name, component, architecture, package_filetype)
+ if package_filetype == 'xz':
+ packages = deb822.Deb822.iter_paragraphs(lzma.decompress(requests.get(packages_url).content).decode("utf-8"))
+ elif package_filetype == 'bz2':
+ packages = deb822.Deb822.iter_paragraphs(bz2.decompress(requests.get(packages_url).content).decode("utf-8"))
+ else:
+ raise Exception('Unable to extract file')
+
+
+ for binary_package in packages:
+ source_field = binary_package.get('Source', binary_package['Package']).split()
+ source_package_name = source_field[0]
+
+ try:
+ source_package_version = source_field[1].strip('()')
+ except IndexError:
+ source_package_version = binary_package['Version']
+
+ source_package_key = (release_name, source_package_name, source_package_version)
+
+ if source_package_key not in source_packages:
+ source_packages[source_package_key] = {}
+
+ if binary_package['Package'] not in source_packages[source_package_key]:
+ source_packages[source_package_key][binary_package['Package']] = {}
+
+ source_packages[source_package_key][binary_package['Package']][architecture] = binary_package['Version']
+
+ print("OK")
+ print(" Updating security-tracker data... ", end='')
+
+ self._update_svn_repository()
+ svn_advisories = self._parse_svn_advisories()
+ print("OK")
+
+ # make a set of the advisory IDs which exist on disk but not in the database
+ new_advisories = set(svn_advisories) - set([advisory.upstream_id for advisory in Advisory.objects.filter(source='debian')])
+
+ print(" Found %i new DSAs/DLAs to download" % len(new_advisories))
+
+ for advisory in new_advisories:
+ print(" Downloading %s... " % advisory, end='')
+ search_packages = set()
+ description = svn_advisories[advisory]['description']
+ description = description[0].upper() + description[1:]
+ base_dsa_name = '-'.join(advisory.lower().split('-')[0:2])
+ long_description = dsa_descriptions.get(base_dsa_name, '')
+
+ with transaction.atomic():
+ db_advisory = Advisory(upstream_id=advisory, source="debian", issued=svn_advisories[advisory]['issued'], short_description=description, description=long_description)
+ db_advisory.save()
+ for cve in svn_advisories[advisory]['cves']:
+ db_vulnerability, created = Vulnerability.objects.get_or_create(upstream_id=cve.upper(), defaults={'first_seen': db_advisory.issued})
+ if created:
+ db_vulnerability.save()
+ db_vulnerability.advisories.add(db_advisory)
+ db_vulnerability.save()
+ for package, versions in svn_advisories[advisory]['packages'].items():
+ for release, version in versions.items():
+ # make the source package object
+ db_srcpackage = SourcePackage(advisory=db_advisory, package=package, release=release, safe_version=version)
+ db_srcpackage.save()
+ search_packages.add(package)
+ search_packages.add(version)
+
+
+ # attempt by convoluted means to get the binary packages for that source package
+ try:
+ if (release, package, version) in source_packages: # package is current so in the repo
+ for binary_package_name, binary_package_architectures in source_packages[(release, package, version)].items():
+ for architecture in binary_package_architectures:
+ binversion = source_packages[(release, package, version)][binary_package_name][architecture]
+ db_binpackage = BinaryPackage(source_package=db_srcpackage, advisory=db_advisory, package=binary_package_name, release=release, safe_version=binversion, architecture=architecture)
+ db_binpackage.save()
+ search_packages.add(binary_package_name)
+ search_packages.add(version)
+ else: # package is not latest in the repo, hopefully it's on snapshots.d.o
+ snapshot_url = "%s/mr/package/%s/%s/allfiles" % (self.snapshot_url, package, version)
+ snapshot_response = requests.get(snapshot_url)
+ if snapshot_response.status_code == 404:
+ print('Package not in snapshots either, removing for now and will try again next time')
+ raise IntegrityError("No packages for this advisory yet")
+ snapshot_data = snapshot_response.json()
+ if snapshot_data['version'] != version:
+ raise Exception("snapshots.d.o returned non-matching result")
+
+ for snapshot_binary in snapshot_data['result']['binaries']:
+ snapshot_binary_architectures = [file['architecture'] for file in snapshot_binary['files'] if file['architecture'] in self.architectures]
+ for architecture in snapshot_binary_architectures:
+ db_binpackage = BinaryPackage(source_package=db_srcpackage, advisory=db_advisory, package=snapshot_binary['name'], release=release, safe_version=snapshot_binary['version'], architecture=architecture)
+ db_binpackage.save()
+ search_packages.add(snapshot_binary['name'])
+ search_packages.add(snapshot_binary['version'])
+
+ db_advisory.search_keywords = " ".join(search_packages)
+ db_advisory.save()
+
+ print("OK")
+ except KeyboardInterrupt:
+ raise
+ except:
+ print("could not get binary packages for %s/%s, assuming there are none" % (release, package))
+
+class UbuntuFeed(object):
+ """
+ Syncs the latest additions to the USN JSON file in to the local database.
+ """
+
+ def __init__(self, usn_url=None, cache_location=None, releases=None, architectures=None):
+ self.usn_url = usn_url or 'https://usn.ubuntu.com/usn-db/database.json.bz2'
+ self.cache_location = cache_location or '%s/advisory_cache/usn' % settings.BASE_DIR
+ self.releases = releases or (
+ 'trusty',
+ 'xenial',
+ )
+ self.architectures = architectures or (
+ 'i386',
+ 'amd64',
+ 'all',
+ )
+
+ def _update_json_advisories(self):
+ """
+ Download and decompress the latest USN data from Ubuntu.
+ """
+ try:
+ os.makedirs(self.cache_location)
+ except OSError: # directory already exists
+ pass
+
+ response = requests.get(self.usn_url, stream=True) # the USN list is a bzip'd JSON file of all the current advisories for all supported releases
+ bytes_downloaded = 0
+ with open("%s/incoming-database.json.bz2" % self.cache_location, 'wb') as f:
+ for chunk in response.iter_content(chunk_size=1024):
+ if chunk:
+ f.write(chunk)
+ f.flush()
+ bytes_downloaded += len(chunk)
+
+ if bytes_downloaded < 1500: # sanity check
+ raise Exception("could not download USN feed")
+ else:
+ try:
+ # un-bzip the file using the bz2 library and atomically replace the existing one if this succeeds
+ with open("%s/incoming-database.json" % self.cache_location, 'wb') as decompressed, bz2.BZ2File("%s/incoming-database.json.bz2" % self.cache_location, 'rb') as compressed:
+ for data in iter(lambda : compressed.read(100 * 1024), b''):
+ decompressed.write(data)
+ os.rename("%s/incoming-database.json" % self.cache_location, "%s/database.json" % self.cache_location)
+ except:
+ raise Exception("could not decompress USN feed")
+
+ def _parse_json_advisories(self):
+ """
+ Produce a dictionary representing USN data from the cache file.
+ """
+
+ with open("%s/database.json" % self.cache_location) as usn_list_file:
+ return json.loads(usn_list_file.read())
+
+ @transaction.atomic
+ def update_local_database(self):
+ """
+ Retrieve the latest JSON data, parse it and add any new advisories to the local database.
+ """
+ print(" Downloading JSON data...")
+ self._update_json_advisories()
+ json_advisories = self._parse_json_advisories()
+ new_advisories = set(json_advisories) - set(['-'.join(advisory.upstream_id.split('-')[1:]) for advisory in Advisory.objects.filter(source='ubuntu')])
+
+ print(" Found %i new USNs to process" % len(new_advisories))
+
+
+ for advisory in new_advisories:
+ print(" Processing USN %s... " % advisory, end='')
+
+ search_packages = set()
+
+ try:
+ advisory_data = json_advisories[advisory]
+ db_advisory = Advisory(
+ upstream_id="USN-%s" % advisory,
+ source="ubuntu",
+ issued=datetime.utcfromtimestamp(advisory_data['timestamp']).replace(tzinfo=pytz.utc),
+ description=advisory_data.get('description', None),
+ action=advisory_data.get('action', None),
+ short_description=advisory_data.get('isummary', None)
+ )
+ db_advisory.save()
+
+ for cve in advisory_data.get('cves', []):
+ db_vulnerability, created = Vulnerability.objects.get_or_create(upstream_id=cve.upper(), defaults={'first_seen': db_advisory.issued})
+ if created:
+ db_vulnerability.save()
+ db_vulnerability.advisories.add(db_advisory)
+ db_vulnerability.save()
+
+ for release, release_data in {release:release_data for release, release_data in json_advisories[advisory]['releases'].items() if release in self.releases}.items():
+
+ # Source packages
+ for src_package, src_package_data in release_data['sources'].items():
+ db_srcpackage = SourcePackage(advisory=db_advisory, package=src_package, release=release, safe_version=src_package_data['version'])
+ db_srcpackage.save()
+ search_packages.add(src_package)
+ search_packages.add(src_package_data['version'])
+
+ # Binary packages
+ for bin_package, bin_package_data in release_data['binaries'].items():
+ bin_package_version = bin_package_data['version']
+
+ # Goes through the architectures to see if the binary package is in them
+ for architecture in [architecture for architecture in release_data.get('archs', {'none': 'dummy'}).keys() if architecture in self.architectures]:
+ for url in release_data['archs'][architecture]['urls'].keys():
+
+ # If binary package is in architecture, add package to db
+ if bin_package == url.split('/')[-1].split('_')[0]:
+ # Adds a binary package in db for each architecture
+ db_binpackage = BinaryPackage(advisory=db_advisory, package=bin_package, release=release, safe_version=bin_package_version, architecture=architecture)
+ db_binpackage.save()
+ search_packages.add(bin_package)
+ search_packages.add(bin_package_version)
+
+ db_advisory.search_keywords = " ".join(search_packages)
+ db_advisory.save()
+ except:
+ print("Error")
+ raise
+ else:
+ print("OK")
+
+class Command(BaseCommand):
+ help = 'Update all sources of advisories'
+
+ def handle(self, *args, **options):
+ self.stdout.write(self.style.MIGRATE_HEADING("Updating DSAs..."))
+ feed = DebianFeed()
+ feed.update_local_database()
+
+ self.stdout.write(self.style.MIGRATE_HEADING("Updating DLAs..."))
+ feed = DebianFeed(cache_location='%s/advisory_cache/dla' % settings.BASE_DIR, list_location='data/DLA/list')
+ feed.update_local_database()
+
+ self.stdout.write(self.style.MIGRATE_HEADING("Updating USNs..."))
+ feed = UbuntuFeed()
+ feed.update_local_database()
+
+ with open("%s/advisory_cache/timestamp" % settings.BASE_DIR, 'w') as timestamp:
+ timestamp.write(str(int(time.time())))
diff --git a/advisories/models.py b/advisories/models.py
new file mode 100644
index 0000000..e1863bf
--- /dev/null
+++ b/advisories/models.py
@@ -0,0 +1,143 @@
+# Copyright (c) 2017 Catalyst.net Ltd
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: Michael Fincham <michael.fincham@catalyst.net.nz>
+# Author: Filip Vujicic <filip.vujicic@catalyst.net.nz>
+# Author: Sam Banks <sam.banks@catalyst.net.nz>
+
+import apt_pkg
+apt_pkg.init_system()
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db import models
+from django.db.models.signals import post_save, pre_delete
+from django.dispatch import receiver
+from django.utils import timezone
+
+class Advisory(models.Model):
+ """
+ "Lowest common denominator" across all vendor advisories.
+ """
+
+ upstream_id = models.CharField(max_length=200, verbose_name="Upstream ID", help_text="The ID used by the vendor to refer to this advisory")
+ short_description = models.CharField(max_length=200, null=True, help_text="One-line description of the advisory")
+ description = models.TextField(null=True, help_text="Longer description of the advisory")
+ action = models.TextField(null=True, help_text="What, if any, actions need to be taken to address the advisory")
+ issued = models.DateTimeField(default=timezone.now, help_text="Date and time at which the advisory was issued")
+ source = models.CharField(choices=settings.ADVISORY_SOURCES, max_length=32, help_text="Vendor source of the advisory")
+ search_keywords = models.TextField(blank=True, null=True, help_text="Space separated list of keywords used to speed up search")
+
+ class Meta:
+ verbose_name_plural = "advisories"
+ ordering = ["-issued"]
+
+ def __str__(self):
+ return self.upstream_id
+
+ def get_absolute_url(self):
+ from django.core.urlresolvers import reverse
+ return reverse('advisory_detail', args=(self.upstream_id, ))
+
+ def source_package_names(self):
+ return ", ".join([package.__str__() for package in self.sourcepackage_set.all()])
+
+ def vulnerability_names(self):
+ return ", ".join([vulnerability.__str__() for vulnerability in self.vulnerabilities.all()])
+
+ def source_url(self):
+ return dict(settings.SOURCE_ADVISORY_DETAIL_URLS)[self.source] % self.upstream_id
+
+
+class SourcePackage(models.Model):
+ """
+ Source package to which an advisory refers. These are not of a direct concern to hosts, as source packages are not actually "installed".
+
+ For Debian advisories, the source package is used to determine what binary packages (and their versions) are considered safe.
+ """
+
+ advisory = models.ForeignKey(Advisory, help_text="Advisory to which this package belongs")
+ package = models.CharField(max_length=200, help_text="Name of source package")
+ release = models.CharField(choices=settings.RELEASES, max_length=32, help_text="Specific release to which this package belongs")
+ safe_version = models.CharField(max_length=200, help_text="Package version that is to be considered 'safe' at the issue of this advisory")
+
+ class Meta:
+ verbose_name_plural = "source packages"
+ ordering = ["-package"]
+
+ def __str__(self):
+ safe_version = self.safe_version
+
+ if self.safe_version == '0':
+ safe_version = ''
+
+ return "%s %s (%s)" % (self.package, safe_version, self.release)
+
+ def source_url(self):
+ return dict(settings.SOURCE_PACKAGE_DETAIL_URLS)[self.advisory.source] % (self.release, self.package)
+
+ def latest_advisory(self):
+ all_advisories = {package.advisory_id for package in SourcePackage.objects.filter(package=self.package, release=self.release)}
+ return Advisory.objects.filter(id__in=all_advisories).order_by('-issued')[0]
+
+class BinaryPackage(models.Model):
+ """
+ Binary package to which an advisory refers.
+
+ In the case of Ubuntu, these are resolved directly from the supplied JSON data. For Debian these will be generated based on the source packages
+ associated with this advisory.
+
+ If source_package is null it is because this binary package was created directly from external data, rather than being generated locally.
+ """
+
+ advisory = models.ForeignKey(Advisory, help_text="Advisory to which this package belongs")
+ source_package = models.ForeignKey(SourcePackage, blank=True, null=True, help_text="If set, the source package from which this binary package was generated")
+ package = models.CharField(max_length=200, help_text="Name of binary package")
+ release = models.CharField(choices=settings.RELEASES, max_length=32, help_text="Specific release to which this package belongs")
+ safe_version = models.CharField(max_length=200, null=True, help_text="Package version that is to be considered 'safe' at the issue of this advisory")
+ architecture = models.CharField(max_length=200, null=True, help_text="Machine architecture")
+
+ class Meta:
+ verbose_name_plural = "binary packages"
+ ordering = ["-package"]
+
+ def __str__(self):
+ if self.safe_version:
+ return "%s %s (%s, %s)" % (self.package, self.safe_version, self.release, self.architecture)
+ else:
+ return "%s (%s, %s)" % (self.package, self.release, self.architecture)
+
+ def source_url(self):
+ return dict(settings.SOURCE_PACKAGE_DETAIL_URLS)[self.advisory.source] % (self.release, self.package)
+
+class Vulnerability(models.Model):
+ """
+ CVE from the MITRE database to allow cross-referencing of advisories.
+ """
+
+ advisories = models.ManyToManyField(Advisory, related_name='vulnerabilities')
+ first_seen = models.DateTimeField(default=timezone.now, help_text="Date and time at which the advisory was issued")
+ upstream_id = models.CharField(max_length=200, help_text="MITRE name of CVE")
+
+ class Meta:
+ verbose_name_plural = "vulnerabilities"
+
+ def __str__(self):
+ return self.upstream_id
+
+ def mitre_url(self):
+ return "https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s" % str(self)
+
+ def source_list(self):
+ return ", ".join(sorted([str(advisory).capitalize() for advisory in self.advisories.values_list('source', flat=True).distinct().order_by()]))
diff --git a/advisorybrowser/__init__.py b/advisorybrowser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/advisorybrowser/__init__.py
diff --git a/advisorybrowser/settings.py b/advisorybrowser/settings.py
new file mode 100644
index 0000000..92e4ea2
--- /dev/null
+++ b/advisorybrowser/settings.py
@@ -0,0 +1,168 @@
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'NOT TODAY'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+ 'django.contrib.admin',
+ 'django.contrib.auth',
+ 'django.contrib.contenttypes',
+ 'django.contrib.sessions',
+ 'django.contrib.messages',
+ 'django.contrib.staticfiles',
+ 'advisories',
+ 'browser',
+]
+
+MIDDLEWARE = [
+ 'django.middleware.security.SecurityMiddleware',
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'advisorybrowser.urls'
+
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [],
+ 'APP_DIRS': True,
+ 'OPTIONS': {
+ 'context_processors': [
+ 'django.template.context_processors.debug',
+ 'django.template.context_processors.request',
+ 'django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages',
+ ],
+ },
+ },
+]
+
+WSGI_APPLICATION = 'advisorybrowser.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.mysql',
+ 'NAME': 'advisories',
+ 'USER': 'advisories',
+ 'PASSWORD': 'NOT TODAY',
+ 'HOST': '127.0.0.1',
+ 'PORT': '3306',
+ 'OPTIONS': {'init_command': "SET sql_mode='STRICT_TRANS_TABLES'"},
+ }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+ {
+ 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+ },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.11/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'Pacific/Auckland'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.11/howto/static-files/
+
+STATIC_URL = '/static/'
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 65535
+
+FIX_REASONS = (
+ ('removed', 'Package removed'),
+)
+
+# Imported from patch-friend...
+
+ADVISORY_SOURCES = (
+ ('ubuntu', 'Ubuntu'),
+ ('debian', 'Debian'),
+)
+
+# For advisories that have been triaged by a human
+ADVISORY_SEVERITIES = (
+ (0, 'Undecided'),
+ (1, 'Low'),
+ (2, 'Standard'),
+ (3, 'High'),
+ (4, 'Critical'),
+)
+
+ADVISORY_SEVERITY_CLASSES = (
+ (0, ''),
+ (1, 'text-muted'),
+ (2, 'text-info'),
+ (3, 'text-warning'),
+ (4, 'text-danger'),
+)
+
+# Current stable releases
+RELEASES = (
+ ('jessie', 'Debian Jessie 8.0'),
+ ('precise', 'Ubuntu Precise LTS 12.04'),
+ ('trusty', 'Ubuntu Trusty LTS 14.04',)
+)
+
+# Data source plugins
+DATA_SOURCES = (
+ ('hostinfo', 'hostinfo'),
+)
+
+SOURCE_ADVISORY_DETAIL_URLS = {
+ ('ubuntu', 'http://www.ubuntu.com/usn/%s/'),
+ ('debian', 'https://security-tracker.debian.org/tracker/%s'),
+}
+
+SOURCE_PACKAGE_DETAIL_URLS = {
+ ('ubuntu', 'http://packages.ubuntu.com/%s/%s'),
+ ('debian', 'https://packages.debian.org/%s/%s'),
+}
+
+APTGET_COMMAND_STUB = 'sudo apt --only-upgrade install'
diff --git a/advisorybrowser/urls.py b/advisorybrowser/urls.py
new file mode 100644
index 0000000..21af84b
--- /dev/null
+++ b/advisorybrowser/urls.py
@@ -0,0 +1,7 @@
+from django.conf.urls import include, url
+from django.contrib import admin
+
+urlpatterns = [
+ url(r'^admin/', admin.site.urls),
+ url(r'', include('browser.urls')),
+]
diff --git a/advisorybrowser/wsgi.py b/advisorybrowser/wsgi.py
new file mode 100644
index 0000000..a6c9e9a
--- /dev/null
+++ b/advisorybrowser/wsgi.py
@@ -0,0 +1,16 @@
+"""
+WSGI config for advisorybrowser project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "advisorybrowser.settings")
+
+application = get_wsgi_application()
diff --git a/browser/__init__.py b/browser/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/browser/__init__.py
diff --git a/browser/admin.py b/browser/admin.py
new file mode 100644
index 0000000..8c38f3f
--- /dev/null
+++ b/browser/admin.py
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
diff --git a/browser/apps.py b/browser/apps.py
new file mode 100644
index 0000000..479a5ac
--- /dev/null
+++ b/browser/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class BrowserConfig(AppConfig):
+ name = 'browser'
diff --git a/browser/models.py b/browser/models.py
new file mode 100644
index 0000000..71a8362
--- /dev/null
+++ b/browser/models.py
@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.
diff --git a/browser/templates/browser/advisory.html b/browser/templates/browser/advisory.html
new file mode 100644
index 0000000..3b6cc34
--- /dev/null
+++ b/browser/templates/browser/advisory.html
@@ -0,0 +1,116 @@
+{% extends 'browser/base.html' %}
+
+{% load advisory_fields %}
+
+{% block title %}{{ object.source|advisory_source }} advisory {{ object.upstream_id }}{% endblock %}
+
+
+{% block content %}
+{% regroup object.sourcepackage_set.all by release as release_list %}
+<table class="horizontal-table">
+ <tr>
+ <th>&nbsp;</th><td><h2>{{ object.upstream_id }}: {{ object.short_description }}</h2></td>
+ </tr>
+ <tr>
+ <th>&nbsp;</th><td><a href="{% url 'index' %}">← Return to advisory browser</a></td>
+ </tr>
+ <tr class="separate-from-above">
+ <th>Source</th><td>{{ object.source|advisory_source }}</td>
+ </tr>
+ <tr>
+ <th>Upstream ID</th><td><a href="{{ object.source_url }}">{{ object.upstream_id }}</a></td>
+ </tr>
+ <tr>
+ <th>Issued</th><td>{{ object.issued }}</td>
+ </tr>
+ <tr>
+ <th>Releases</th>
+ <td>
+ {% for release in release_list %}
+ {{ release.grouper }}{% if not forloop.last %}, {% endif %}
+ {% endfor %}
+ </td>
+ </tr>
+
+ <tr class="separate-from-above">
+ <th>Description</th><td>{{ advisory.description|ignore_none|paragraphbreaks }}</td>
+ </tr>
+ <tr>
+ <th>CVEs</th><td>{% for vuln in advisory.vulnerabilities.all %}<a href="{% url 'vulnerability_detail' vuln.upstream_id %}">{{ vuln.upstream_id }}</a>{% if not forloop.last %}, {% endif %}{% endfor %}</td>
+ </tr>
+
+ <tr>
+ <th>Required action</th><td>{{ advisory.action|ignore_none|paragraphbreaks }}</td>
+ </tr>
+ <tr class="separate-from-above">
+ <th class="table-label">Source packages</th>
+ <td>
+ <table class="table package-table">
+ {% for release in release_list %}
+ <thead>
+ <tr class="release-name{% if forloop.first %}-first{% endif %}">
+ <th colspan="2">{{ release.grouper|advisory_source }}</th>
+ <tr>
+ <th class="package-name">Package</th><th>Safe version</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for item in release.list %}
+ <tr>
+ <td class="package-name"><a href="{{ item.source_url }}">{{ item.package }}</a></td><td>{{ item.safe_version }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ {% endfor %}
+ </table>
+ </td>
+ </tr>
+ <tr class="separate-from-above">
+ <th class="table-label">Binary packages</th>
+ <td>
+ <table class="table package-table">
+ {% for release_name, release in binary_packages.items %}
+ <thead>
+ <tr class="release-name{% if forloop.first %}-first{% endif %}">
+ <th colspan="3">{{ release_name|advisory_source }}</th>
+ <tr>
+ <th class="package-name">Package</th><th>Safe version</th><th>Architectures</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for key, package_data in release.items %}
+ <tr>
+ <td class="package-name"><a href="{{ package_data.package.source_url }}">{{ package_data.package.package }}</a></td><td>{{ package_data.package.safe_version }}</td><td>{{ package_data.architectures|sortedlist }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ {% endfor %}
+ </table>
+ </td>
+ </tr>
+ <tr class="separate-from-above">
+ <th class="table-label">Update commands</th>
+ <td>
+ <table class="table package-table">
+ <thead>
+ <tr class="release-name-first">
+ <th class="package-name">Release</th><th>Command</th>
+ </tr>
+ </thead>
+ {% for release_name, release in binary_packages.items %}
+ <tbody>
+ <tr>
+ <td class="package-name">{{ release_name }}</td>
+ <td>
+ <div class="card" style="padding: 10px;">
+ <p class="card-text">{{ aptget_command }} {% for key, package_data in release.items %} {{ package_data.package.package }} {% endfor %}</p>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+ {% endfor %}
+ </table>
+ </td>
+ </tr>
+ </table>
+ {% endblock %}
diff --git a/browser/templates/browser/base.html b/browser/templates/browser/base.html
new file mode 100644
index 0000000..4524e8d
--- /dev/null
+++ b/browser/templates/browser/base.html
@@ -0,0 +1,76 @@
+<!doctype html>
+<html>
+<head>
+ <title>{% block title %}Advisory browser{% endblock %}</title>
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta.2/css/bootstrap.min.css" integrity="sha384-PsH8R72JQ3SOdhVi3uxftmaW6Vc51MKb0q5P2rRUpPvrszuE4W1povHYgTpBfshb" crossorigin="anonymous">
+ <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.16/css/dataTables.bootstrap4.min.css">
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+ <style>
+ a {
+ color: #135069 !important;
+ }
+
+ .date {
+ width: 100px;
+ }
+
+ .date, th {
+ white-space: nowrap;
+ }
+
+ th {
+ vertical-align: top;
+ }
+
+ table.horizontal-table th {
+ padding-right: 10px;
+ text-align: right;
+ }
+
+ table.horizontal-table table.table th {
+ text-align: left;
+ }
+
+ .separate-from-above th, .separate-from-above td {
+ padding-top: 15px;
+ }
+
+ th.table-label {
+ padding-top: 31px;
+ }
+
+ .page-item.active .page-link {
+ background-color: #d5d5d5 !important;
+ border-color: #135069 !important;
+ }
+
+ #maintable td:nth-child(6), #maintable td:nth-child(1) {
+ white-space: nowrap;
+ }
+
+ .ellipsis {
+ position: relative;
+ }
+
+ .ellipsis:before {
+ content: '&nbsp;';
+ visibility: hidden;
+ }
+
+ .ellipsis span {
+ position: absolute;
+ left: 0;
+ right: 0;
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ }
+ </style>
+</head>
+<body>
+ <div class="container-fluid">
+ {% block content %}{% endblock %}
+ </div>
+</body>
+</html>
diff --git a/browser/templates/browser/cves.html b/browser/templates/browser/cves.html
new file mode 100644
index 0000000..8e26d0f
--- /dev/null
+++ b/browser/templates/browser/cves.html
@@ -0,0 +1,49 @@
+{% extends 'browser/base.html' %}
+
+{% block title %}CVE browser{% endblock %}
+
+{% block content %}
+<h1>CVE browser</h1>
+<div class="row">
+ <div class="col">
+ <p>There are currently {{ vulnerability_count }} CVEs known to this system. Information last updated {{ updated }}.</p>
+ <p>Because Ubuntu do not issue advisories for packages in the "universe" component, there tend to be more CVEs associated with Debian advisories.</p>
+ </div>
+</div>
+<div class="row">
+ <div class="col">
+ <h2>All CVEs</h2>
+ </div>
+ <table id="maintable" class="table" width="100%">
+ <thead>
+ <tr>
+ <th style="max-width: 100px;">Upstream ID</th><th>Advisories</th><th>Fixed in</th><th style="max-width: 100px">First seen</th>
+ </tr>
+ </thead>
+ </table>
+</div>
+
+<script src="https://code.jquery.com/jquery-3.2.1.min.js" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.3/umd/popper.min.js" integrity="sha384-vFJXuSJphROIrBnz7yo7oB41mKfc8JzQZiCq4NCceLEaO4IHwicKwpJf9c9IpFgh" crossorigin="anonymous"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta.2/js/bootstrap.min.js" integrity="sha384-alpBpkh1PFOepccYVYDB4do5UnbKysX5WZXm3XxPqe5iKTfUKjNkCk9SaVuEZflJ" crossorigin="anonymous"></script>
+<script type="text/javascript" language="javascript" src="https://cdn.datatables.net/1.10.16/js/jquery.dataTables.min.js"></script>
+<script type="text/javascript" language="javascript" src="https://cdn.datatables.net/1.10.16/js/dataTables.bootstrap4.min.js"></script>
+<script type="text/javascript" class="init">
+ $(document).ready(function() {
+ $('#maintable').DataTable( {
+ "processing": true,
+ "serverSide": true,
+ "order": [[ 3, "desc" ]],
+ "ajax": "{% url 'cve_table' %}",
+ "columns": [
+ null,
+ { "orderable": false },
+ { "orderable": false },
+ null
+ ]
+ });
+ });
+</script>
+<br>
+<p>More information will be added to this application soon! For enquiries contact <a href="mailto:michael@hotplate.co.nz">michael@hotplate.co.nz</a></p>
+{% endblock %}
diff --git a/browser/templates/browser/index.html b/browser/templates/browser/index.html
new file mode 100644
index 0000000..95418fb
--- /dev/null
+++ b/browser/templates/browser/index.html
@@ -0,0 +1,82 @@
+{% extends 'browser/base.html' %}
+
+{% block content %}
+<h1>Advisory browser</h1>
+<div class="row">
+ <div class="col">
+ <p>There are currently {{ advisory_count }} advisories known to this system. Information last updated {{ updated }}.</p>
+ </div>
+</div>
+<div class="row">
+ <div class="col-sm">
+ <table class="table table-sm"><thead><tr><th colspan="2">Recent advisories</th></tr>
+ </thead><tbody>
+ {% for advisory in recent_advisories %}
+ <tr><td class="date">{{ advisory.issued|date:"Y-m-d" }}</td><td class="ellipsis"><span><a href="{% url 'advisory_detail' advisory.upstream_id %}">{{ advisory.upstream_id }}</a> {{ advisory.short_description }}</span></td></tr>
+ {% endfor %}
+ </table>
+ </div>
+ <div class="col-sm">
+ <table class="table table-sm"><thead><tr><th colspan="2">Recent packages</th></tr></thead><tbody>
+ {% for package in recent_packages %}
+ <tr><td class="date">{{ package.advisory.issued|date:"Y-m-d" }}</td><td class="ellipsis"><span><a href="{% url 'advisory_detail' package.advisory.upstream_id %}">{{ package.package }} {{ package.safe_version }}</a></span></td></tr>
+ {% endfor %}
+ </tbody>
+ </table>
+</div>
+<div class="col-sm">
+ <table class="table table-sm">
+ <thead>
+ <tr>
+ <th colspan="2"><a href="{% url 'cves' %}">Recent CVEs</a></th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for vulnerability in recent_vulnerabilities %}
+ <tr>
+ <td class="date">{{ vulnerability.first_seen|date:"Y-m-d" }}</td><td class="ellipsis"><span><a href="{% url 'vulnerability_detail' vulnerability.upstream_id %}">{{ vulnerability.upstream_id }}</a> from {{ vulnerability.source_list }}</span></td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+</div>
+</div>
+<div class="row">
+ <div class="col">
+ <h2>All advisories</h2>
+ </div>
+ <table id="maintable" class="table" width="100%">
+ <thead>
+ <tr>
+ <th>Upstream ID</th><th>Short description</th><th>Source package names</th><th>CVEs</th><th>Source</th><th>Issued</th>
+ </tr>
+ </thead>
+ </table>
+</div>
+
+<script src="https://code.jquery.com/jquery-3.2.1.min.js" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.3/umd/popper.min.js" integrity="sha384-vFJXuSJphROIrBnz7yo7oB41mKfc8JzQZiCq4NCceLEaO4IHwicKwpJf9c9IpFgh" crossorigin="anonymous"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta.2/js/bootstrap.min.js" integrity="sha384-alpBpkh1PFOepccYVYDB4do5UnbKysX5WZXm3XxPqe5iKTfUKjNkCk9SaVuEZflJ" crossorigin="anonymous"></script>
+<script type="text/javascript" language="javascript" src="https://cdn.datatables.net/1.10.16/js/jquery.dataTables.min.js"></script>
+<script type="text/javascript" language="javascript" src="https://cdn.datatables.net/1.10.16/js/dataTables.bootstrap4.min.js"></script>
+<script type="text/javascript" class="init">
+ $(document).ready(function() {
+ $('#maintable').DataTable( {
+ "processing": true,
+ "serverSide": true,
+ "order": [[ 5, "desc" ]],
+ "ajax": "{% url 'table' %}",
+ "columns": [
+ null,
+ { "orderable": false },
+ { "orderable": false },
+ { "orderable": false },
+ null,
+ null
+ ]
+ });
+ });
+</script>
+<br>
+<p>More information will be added to this application soon! For enquiries contact <a href="mailto:michael@hotplate.co.nz">michael@hotplate.co.nz</a></p>
+{% endblock %}
diff --git a/browser/templates/browser/vulnerability.html b/browser/templates/browser/vulnerability.html
new file mode 100644
index 0000000..dd16297
--- /dev/null
+++ b/browser/templates/browser/vulnerability.html
@@ -0,0 +1,37 @@
+{% extends 'browser/base.html' %}
+
+{% load advisory_fields %}
+
+{% block title %}{{ vulnerability.upstream_id }}{% endblock %}
+
+
+{% block content %}
+ <table class="horizontal-table">
+ <tr>
+ <th>&nbsp;</th><td><h2>{{ vulnerability.upstream_id }}</h2></td>
+ </tr>
+ <tr>
+ <th>&nbsp;</th><td><a href="{% url 'index' %}">← Return to advisory browser</a></td>
+ </tr>
+ <tr class="separate-from-above">
+ <th>Upstream ID</th><td><a href="https://cve.mitre.org/cgi-bin/cvename.cgi?name={{ vulnerability.upstream_id }}">{{ vulnerability.upstream_id }}</a> (<a href="https://security-tracker.debian.org/tracker/{{ vulnerability.upstream_id }}">Debian</a>, <a href="https://people.canonical.com/~ubuntu-security/cve/?cve={{ vulnerability.upstream_id }}">Ubuntu</a>)</td>
+ </tr>
+ <tr>
+ <th>First seen</th><td>{{ vulnerability.first_seen }}</td>
+ </tr>
+
+ <tr class="separate-from-above">
+ <th class="table-label">Advisories</th><td>
+
+ <table class="table package-table">
+ <thead>
+ <tr class="release-name-first">
+ <th class="package-name">Upstream ID</th><th>Short description</th><th>Source package names</th><th>Issued</th>
+ </tr>
+ </thead>
+
+
+ <tbody>{% for advisory in vulnerability.advisories.all %}<tr><td><a href="{% url 'advisory_detail' advisory.upstream_id %}">{{ advisory.upstream_id }}</a></td><td>{{advisory.short_description }}</td><td>{{advisory.source_package_names}}</td><td>{{ advisory.issued }}</td></tr>{% endfor %}</tbody></table></td>
+ </tr>
+ </table>
+{% endblock %}
diff --git a/browser/templatetags/__init__.py b/browser/templatetags/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/browser/templatetags/__init__.py
diff --git a/browser/templatetags/advisory_fields.py b/browser/templatetags/advisory_fields.py
new file mode 100644
index 0000000..28d5a50
--- /dev/null
+++ b/browser/templatetags/advisory_fields.py
@@ -0,0 +1,45 @@
+import textwrap
+import re
+
+from django import template
+from django.conf import settings
+from django.template.defaultfilters import stringfilter
+from django.utils.html import conditional_escape
+from django.utils.safestring import mark_safe
+from django.core.urlresolvers import reverse
+
+register = template.Library()
+
+@register.filter
+@stringfilter
+def advisory_source(value):
+ return value.capitalize()
+
+@register.filter
+@stringfilter
+def ignore_none(value):
+ if value == 'None':
+ return ''
+
+ return value
+
+@register.filter
+def sortedlist(value):
+ return ", ".join(sorted(value))
+
+@register.filter(needs_autoescape=True)
+@stringfilter
+def paragraphbreaks(value, autoescape=True):
+ if autoescape:
+ esc = conditional_escape
+ else:
+ esc = lambda x: x
+
+ value = textwrap.dedent(value).replace('\r\n', '\n') # fix windows linebreaks
+
+ result = '<p>%s</p>' % '</p><p>'.join(esc(value).split('\n\n'))
+ result = re.sub(r"(CVE-[0-9]+-[0-9]+)", r'<a href="%s\1">\1</a>' % reverse('vulnerability_detail', args=['']), result)
+ result = re.sub(r"(DSA-[0-9]+-[0-9]+)", r'<a href="%s\1">\1</a>' % reverse('advisory_detail', args=['']), result)
+ result = re.sub(r"(DLA-[0-9]+-[0-9]+)", r'<a href="%s\1">\1</a>' % reverse('advisory_detail', args=['']), result)
+ result = re.sub(r"(USN-[0-9]+-[0-9]+)", r'<a href="%s\1">\1</a>' % reverse('advisory_detail', args=['']), result)
+ return mark_safe(result)
diff --git a/browser/urls.py b/browser/urls.py
new file mode 100644
index 0000000..dd52d00
--- /dev/null
+++ b/browser/urls.py
@@ -0,0 +1,12 @@
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+ url(r'^$', views.index, name='index'),
+ url(r'^table$', views.AdvisoryTableView.as_view(), name='table'),
+ url(r'^cve_table$', views.VulnerabilityTableView.as_view(), name='cve_table'),
+ url(r'^advisory/(.*)$', views.advisory, name='advisory_detail'),
+ url(r'^vulnerability/(.*)$', views.vulnerability, name='vulnerability_detail'),
+ url(r'^cves$', views.cves, name='cves'),
+]
diff --git a/browser/views.py b/browser/views.py
new file mode 100644
index 0000000..2cdf548
--- /dev/null
+++ b/browser/views.py
@@ -0,0 +1,108 @@
+from django.shortcuts import render, get_object_or_404
+from django.http import HttpResponse
+
+from advisories.models import *
+from django_datatables_view.base_datatable_view import BaseDatatableView
+
+from django.core.urlresolvers import reverse
+from django.utils import formats
+from django.conf import settings
+from django.db.models import Q
+
+import datetime
+import collections
+
+def index(request):
+ try:
+ with open("%s/advisory_cache/timestamp" % settings.BASE_DIR, 'r') as timestamp:
+ updated = str(datetime.datetime.fromtimestamp(int(timestamp.readline()))).strip('.')
+ except:
+ updated = "never"
+ advisory_count = Advisory.objects.all().count()
+ recent_advisories = Advisory.objects.order_by('-issued').all()[0:5]
+ recent_packages = SourcePackage.objects.order_by('-advisory__issued').all()[0:5]
+ recent_vulnerabilities = Vulnerability.objects.order_by('-first_seen').all()[0:5]
+ return render(request, 'browser/index.html', {'recent_advisories': recent_advisories, 'recent_packages': recent_packages, 'recent_vulnerabilities': recent_vulnerabilities, 'advisory_count': advisory_count, 'updated': updated})
+
+def cves(request):
+ try:
+ with open("%s/advisory_cache/timestamp" % settings.BASE_DIR, 'r') as timestamp:
+ updated = str(datetime.datetime.fromtimestamp(int(timestamp.readline()))).strip('.')
+ except:
+ updated = "never"
+ vulnerability_count = Vulnerability.objects.all().count()
+ return render(request, 'browser/cves.html', {'vulnerability_count': vulnerability_count, 'updated': updated})
+
+def advisory(request, upstream_id):
+ advisory = get_object_or_404(Advisory, upstream_id=upstream_id)
+
+ # XXX this seems like a horrible way to do this but the triple nested regroup in the template didn't work
+
+ binary_packages = collections.defaultdict(dict)
+
+ for package in advisory.binarypackage_set.all():
+ package_key = "%s %s" % (package.package, package.safe_version)
+
+ if package_key not in binary_packages[package.release]:
+ binary_packages[package.release][package_key] = {'package': package, 'architectures': []}
+
+ binary_packages[package.release][package_key]['architectures'].append(package.architecture)
+
+ return render(request, 'browser/advisory.html', {'object': advisory, 'advisory': advisory, 'binary_packages': dict(binary_packages), 'aptget_command': 'apt --only-upgrade install', })
+
+def vulnerability(request, upstream_id):
+ vulnerability = get_object_or_404(Vulnerability, upstream_id=upstream_id)
+
+ return render(request, 'browser/vulnerability.html', {'vulnerability': vulnerability})
+
+class AdvisoryTableView(BaseDatatableView):
+ model = Advisory
+ order_columns = ['upstream_id', '', '', '', 'source', 'issued']
+ max_display_length = 500
+
+ def filter_queryset(self, qs):
+ search = self.request.GET.get(u'search[value]', None)
+ if search:
+ if search.upper().startswith('CVE-'):
+ vuln = Vulnerability.objects.filter(upstream_id__istartswith=search.upper())
+ qs = qs.filter(vulnerabilities__in=vuln)
+ else:
+ qs = qs.filter(search_keywords__icontains=search)
+ return qs
+
+ def prepare_results(self, qs):
+ json_data = []
+ for item in qs:
+ json_data.append([
+ '<a href="%s">%s</a>' % (reverse('advisory_detail', args=[item.upstream_id]), item.upstream_id),
+ item.short_description,
+ item.source_package_names(),
+ ", ".join(sorted(['<a href="%s">%s</a>' % (reverse('vulnerability_detail', args=[vulnerability.upstream_id]), vulnerability.__str__()) for vulnerability in item.vulnerabilities.all()])),
+ item.source.capitalize(),
+ formats.date_format(item.issued, "Y-m-d")
+ ])
+ return json_data
+
+class VulnerabilityTableView(BaseDatatableView):
+ model = Vulnerability
+ order_columns = ['upstream_id', '', '', 'first_seen']
+ max_display_length = 500
+
+ def filter_queryset(self, qs):
+ search = self.request.GET.get(u'search[value]', None)
+ if search:
+ qs = qs.filter(Q(upstream_id__icontains=search) | Q(advisories__source__in=[search]) | Q(advisories__upstream_id__in=[search]))
+ return qs
+
+ def prepare_results(self, qs):
+ json_data = []
+ for item in qs:
+ json_data.append([
+ '<a href="%s">%s</a>' % (reverse('vulnerability_detail', args=[item.upstream_id]), item.upstream_id),
+ ", ".join(sorted(['<a href="%s">%s</a>' % (reverse('advisory_detail', args=[advisory.upstream_id]), advisory.__str__()) for advisory in item.advisories.all()])),
+ item.source_list(),
+ formats.date_format(item.first_seen, "Y-m-d")
+ ])
+ return json_data
+
+
diff --git a/manage.py b/manage.py
new file mode 100755
index 0000000..776be6a
--- /dev/null
+++ b/manage.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "advisorybrowser.settings")
+ try:
+ from django.core.management import execute_from_command_line
+ except ImportError:
+ # The above import may fail for some other reason. Ensure that the
+ # issue is really that Django is missing to avoid masking other
+ # exceptions on Python 2.
+ try:
+ import django
+ except ImportError:
+ raise ImportError(
+ "Couldn't import Django. Are you sure it's installed and "
+ "available on your PYTHONPATH environment variable? Did you "
+ "forget to activate a virtual environment?"
+ )
+ raise
+ execute_from_command_line(sys.argv)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d057b5a
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,15 @@
+beautifulsoup4==4.6.0
+certifi==2017.11.5
+chardet==3.0.4
+Django==1.11.7
+django-datatables-view==1.14.0
+idna==2.6
+nose==1.3.7
+pkg-resources==0.0.0
+python-dateutil==2.6.1
+python-debian==0.1.31
+pytz==2017.3
+requests==2.18.4
+six==1.11.0
+svn==0.3.45
+urllib3==1.22