'''cr_lint.py: lint checks'''
#
# Copyright (C) 2013-2015 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function
from apt import apt_pkg
from debian.deb822 import Deb822
import glob
import os
import re
import stat
import yaml

from clickreviews.frameworks import Frameworks
from clickreviews.cr_common import (
    ClickReview,
)
from clickreviews.common import (
    open_file_read,
    cmd,
    error,
)
from clickreviews.common import (
    find_external_symlinks,
)

CONTROL_FILE_NAMES = ["control", "manifest", "preinst"]
MINIMUM_CLICK_FRAMEWORK_VERSION = "0.4"


class ClickReviewLint(ClickReview):
    '''This class represents click lint reviews'''

    def __init__(self, fn, overrides=None):
        '''Set up the class.'''
        ClickReview.__init__(self, fn, "lint", overrides=overrides)
        if not self.is_click and not self.is_snap1:
            return

        if self.is_click and "md5sums" not in CONTROL_FILE_NAMES:
            CONTROL_FILE_NAMES.append("md5sums")
        elif self.is_snap1:
            CONTROL_FILE_NAMES.append("hashes.yaml")
        self.control_files = dict()
        self._list_control_files()
        # Valid values for Architecture in DEBIAN/control. Note:
        #  all    - no compiled code
        #  multi  - fat packages (click manifest declares which ones as a list)
        #  armhf  - compiled, single arch (unity8-based devices)
        #  i386   - compiled, single arch (unity8-based emulator and desktop)
        #  amd64  - compiled, single arch (unity8-based desktop)
        #  arm64  - compiled, single arch (not available yet)
        #  ppc64el - compiled, single arch (not available yet)
        #  powerpc - compiled, single arch (not available yet)
        self.valid_compiled_architectures = ['armhf',
                                             'i386',
                                             'amd64',
                                             'arm64',
                                             ]
        self.valid_control_architectures = ['all',
                                            'multi',
                                            ] + self.valid_compiled_architectures
        self.vcs_files = ['.bzr*',
                          # '.excludes',  # autogenerated by SDK
                          '.git*',
                          '.idea',
                          '.svn*',
                          '.hg',
                          '.project',
                          'CVS*',
                          'RCS*'
                          ]

        if self.manifest is not None and 'maintainer' in self.manifest:
            maintainer = self.manifest['maintainer']
            self.email = maintainer.partition('<')[2].rstrip('>')
            self.is_core_app = \
                (self.click_pkgname.startswith('com.ubuntu.') and
                 not self.click_pkgname.startswith('com.ubuntu.developer.') and
                 (self.email == 'ubuntu-touch-coreapps@lists.launchpad.net' or
                  self.email == 'ubuntu-devel-discuss@lists.ubuntu.com'))
            # "core scope" is not necessarily a word we use right now, but
            # we want to special case scopes which are written through our
            # vetted development process.
            self.is_core_scope = (self.click_pkgname.startswith('com.ubuntu.scopes.') and
                                  self.email == 'ubuntu-devel-discuss@lists.ubuntu.com')
            # "core snappy" is not necessarily a word we use right now, but
            # we want to special case scopes which are written through our
            # vetted development process.
            self.is_core_snappy = \
                (self.click_pkgname.startswith('com.ubuntu.snappy.') and
                 self.email == 'ubuntu-devel-discuss@lists.ubuntu.com')
        else:
            self.email = None
            self.is_core_app = False
            self.is_core_scope = False
            self.is_core_snappy = False

        self._list_all_compiled_binaries()

        self.known_hooks = ['accounts',
                            'account-application',
                            'account-provider',
                            'account-qml-plugin',
                            'account-service',
                            'apparmor',
                            'apparmor-profile',
                            'bin-path',  # obsoleted, ignored
                            'content-hub',
                            'desktop',
                            'pay-ui',
                            'push-helper',
                            'scope',
                            'snappy-systemd',  # obsoleted, ignored
                            'urls']

        self.redflagged_hooks = ['pay-ui',
                                 'apparmor-profile',
                                 ]

        # Valid values for 'type' in packaging yaml
        # - app
        # - framework
        # - kernel
        # - oem (deprecated, 15.04 only)
        # - gadget
        # - os
        self.snappy_valid_types = ['app',
                                   'framework',
                                   'kernel',
                                   'oem',
                                   'gadget',
                                   'os',
                                   ]
        self.snappy_redflagged_types = ['framework',
                                        'kernel',
                                        'oem',
                                        'gadget',
                                        'os',
                                        ]

    def _list_control_files(self):
        '''List all control files with their full path.'''
        for i in CONTROL_FILE_NAMES:
            self.control_files[i] = os.path.join(self.unpack_dir,
                                                 "DEBIAN/%s" % i)

    def check_control_files(self):
        '''Check DEBIAN/* files'''
        if not self.is_click and not self.is_snap1:
            return

        for f in self.control_files:
            t = 'info'
            n = self._get_check_name(
                'DEBIAN_has_files', extra=os.path.basename(f))
            s = "OK"
            if not os.path.isfile(self.control_files[os.path.basename(f)]):
                if self.is_snap1 and os.path.basename(f) == 'md5sums':
                    s = "OK (skip md5sums with snap)"
                else:
                    t = 'error'
                    s = "'%s' not found in DEBIAN/" % os.path.basename(f)
            self._add_result(t, n, s)

        found = []
        for f in sorted(glob.glob("%s/DEBIAN/*" % self.unpack_dir)):
            if os.path.basename(f) not in self.control_files:
                found.append(os.path.basename(f))
        t = 'info'
        n = self._get_check_name('DEBIAN_extra_files')
        s = 'OK'
        if len(found) > 0:
            t = 'warn'
            s = 'found extra files in DEBIAN/: %s' % ", ".join(found)
        self._add_result(t, n, s)

    def check_control(self):
        '''Check control()'''
        if not self.is_click and not self.is_snap1:
            return

        fh = self._extract_control_file()
        tmp = list(Deb822.iter_paragraphs(fh))
        t = 'info'
        n = self._get_check_name('control_structure')
        s = 'OK'
        if len(tmp) != 1:
            self._add_result('error', n,
                             'control malformed: too many paragraphs')
            return
        self._add_result(t, n, s)

        control = tmp[0]
        fields = ['Package',
                  'Version',
                  'Click-Version',
                  'Architecture',
                  'Maintainer',
                  'Installed-Size',
                  'Description']

        error = False
        for f in sorted(fields):
            t = 'info'
            n = self._get_check_name('control_has_field', extra=f)
            s = 'OK'
            if f not in control:
                if f == 'Maintainer' and self.is_snap1:
                    s = 'OK (maintainer not required for snappy)'
                else:
                    t = 'error'
                    s = "'%s' missing" % f
                    error = True
            self._add_result(t, n, s)
        if error is True:
            return

        t = 'info'
        n = self._get_check_name('control_extra_fields')
        s = 'OK'
        found = []
        for k in sorted(control.keys()):
            if k not in fields:
                found.append(k)
        if len(found) > 0:
            self._add_result('error', n,
                             "found extra fields: '%s'" % (", ".join(found)))

        t = 'info'
        n = self._get_check_name('control_package_match')
        s = "OK"
        if self.manifest['name'] != self.click_pkgname:
            t = 'error'
            s = "Package=%s does not match manifest name=%s" % \
                (self.manifest['name'], self.click_pkgname)
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('control_version_match')
        s = "OK"
        if self.manifest['version'] != self.click_version:
            t = 'error'
            s = "Version=%s does not match manifest version=%s" % \
                (self.manifest['version'], self.click_version)
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('control_architecture_match')
        s = 'OK'
        if 'architecture' in self.manifest:
            if control['Architecture'] == "multi":
                if not isinstance(self.manifest['architecture'],
                                  (list, tuple)):
                    t = 'error'
                    s = 'If arch=multi, manifest architecture needs to ' + \
                        'list all the individual architectures.'
                elif list(filter(lambda a:
                                 a not in self.valid_compiled_architectures,
                                 self.manifest['architecture'])):
                    t = 'error'
                    s = 'If arch=multi, manifest architecture needs to ' + \
                        'comprise of only compiled architectures.'
            elif control['Architecture'] != self.manifest['architecture'] and \
                    not self.is_snap1:  # snappy doesn't use this field; ignore
                t = 'error'
                s = "Architecture=%s " % control['Architecture'] + \
                    "does not match manifest architecture=%s" % \
                    self.manifest['architecture']
        else:  # Lack of architecture in manifest is not an error
            t = 'info'
            s = 'OK: architecture not specified in manifest'
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('control_maintainer_match')
        s = 'OK'
        if 'maintainer' in self.manifest:
            if 'Maintainer' in control and \
                    control['Maintainer'] != self.manifest['maintainer']:
                t = 'error'
                s = "Maintainer=%s does not match manifest maintainer=%s" % \
                    (control['Maintainer'], self.manifest['maintainer'])
        else:
            s = 'Skipped: maintainer not in manifest'
        self._add_result(t, n, s)

        # TODO: click currently sets the Description to be the manifest title.
        # Is this intended behavior?
        t = 'info'
        n = self._get_check_name('control_description_match')
        s = 'OK'
        if 'title' in self.manifest:
            if control['Description'].strip() != \
               self.manifest['title'].strip():
                t = 'error'
                s = "Description=%s does not match manifest title=%s" % \
                    (control['Description'], self.manifest['title'])
        else:
            t = 'warn'
            s = 'Skipped: title not in manifest'
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('control_click_version_up_to_date')
        s = 'OK'
        l = None

        if apt_pkg.version_compare(
                control['Click-Version'], MINIMUM_CLICK_FRAMEWORK_VERSION) < 0:
            t = 'error'
            s = "Click-Version is too old, has '%s', needs '%s' or newer" % (
                control['Click-Version'], MINIMUM_CLICK_FRAMEWORK_VERSION)
            l = 'http://askubuntu.com/questions/417366/what-does-lint-control-click-version-up-to-date-mean/417367'
        self._add_result(t, n, s, l)

        t = 'info'
        n = self._get_check_name('control_installed_size')
        s = 'OK'
        try:
            int(control['Installed-Size'])
        except TypeError:
            t = 'error'
            s = "invalid Installed-Size '%s'" % (control['Installed-Size'])
        self._add_result(t, n, s)

    def check_md5sums(self):
        '''Check md5sums()'''
        if not self.is_click:
            return
        curdir = os.getcwd()
        fh = open_file_read(self.control_files["md5sums"])
        badsums = []
        os.chdir(self.unpack_dir)
        for line in fh.readlines():
            split_line = line.strip().split()
            fn = " ".join(split_line[1:])
            (rc, out) = cmd(['md5sum', fn])
            if line != out:
                badsums.append(fn)
        fh.close()
        os.chdir(curdir)

        t = 'info'
        n = self._get_check_name('md5sums')
        s = 'OK'
        if len(badsums) > 0:
            t = 'error'
            s = 'found bad checksums: %s' % ", ".join(badsums)
        self._add_result(t, n, s)

    def check_preinst(self):
        '''Check preinst()'''
        if not self.is_click and not self.is_snap1:
            return

        expected = '''#! /bin/sh
echo "Click packages may not be installed directly using dpkg."
echo "Use 'click install' instead."
exit 1
'''
        fh = open_file_read(self.control_files["preinst"])
        contents = ""
        for line in fh.readlines():
            contents += line
        fh.close()

        t = 'info'
        n = self._get_check_name('preinst')
        s = "OK"
        if contents != expected:
            t = 'error'
            s = "unexpected preinst contents"
        self._add_result(t, n, s)

    def check_hooks(self):
        '''Check click manifest hooks'''
        if not self.is_click and not self.is_snap1:
            return

        #  oem snaps don't have a hooks entry
        if self.is_snap_oem:
            return

        # Some checks are already handled in
        # cr_common.py:_verify_manifest_structure()

        t = 'info'
        n = self._get_check_name('hooks_multiple_apps')
        s = 'OK'
        count = 0
        for app in self.manifest['hooks']:
            if "desktop" in self.manifest['hooks'][app]:
                count += 1
        if count > 1:
            # 'info' for now but this might be removed in a future version
            # (see https://launchpad.net/bugs/1496402)
            t = 'info'
            s = 'more than one desktop app specified in hooks'
        self._add_result(t, n, s)

        # Verify keys are well-formatted
        for app in self.manifest['hooks']:
            t = 'info'
            n = self._get_check_name('hooks_valid', app=app)
            s = "OK"
            if not re.search(r'^[A-Za-z0-9+-.:~-]+$', app):
                t = 'error'
                s = "malformed application name: '%s'" % app
            self._add_result(t, n, s)

        # Verify we have the required hooks
        required = ['apparmor']
        for f in required:
            for app in self.manifest['hooks']:
                t = 'info'
                n = self._get_check_name('hooks', app=app, extra=f)
                s = "OK"
                if f in list(filter(lambda a: a.startswith('account-'),
                   self.known_hooks)):
                    s = "OK (run check-online-accounts for more checks)"
                elif f == "apparmor":
                    s = "OK (run check-security for more checks)"
                elif f == "content-hub":
                    s = "OK (run check-content-hub for more checks)"
                elif f == "desktop":
                    s = "OK (run check-desktop for more checks)"
                elif f == "scope":
                    s = "OK (run check-scope for more checks)"
                elif f == "urls":
                    s = "OK (run check-url-dispatcher for more checks)"

                if f not in self.manifest['hooks'][app]:
                    # TODO: when have apparmor policy for account-provider and
                    #       account-qml-plugin, remove this conditional
                    if f == 'apparmor' and \
                       'apparmor-profile' in self.manifest['hooks'][app]:
                        # Don't require apparmor if have apparmor-profile
                        pass
                    elif f != 'apparmor' or \
                            len(self.manifest['hooks'][app]) != 2 or \
                            'account-provider' not in \
                            self.manifest['hooks'][app] or \
                            'account-qml-plugin' not in \
                            self.manifest['hooks'][app]:
                        t = 'error'
                        s = "'%s' hook not found for '%s'" % (f, app)
                self._add_result(t, n, s)

        mutually_exclusive = ['scope', 'desktop']
        for app in self.manifest['hooks']:
            t = 'info'
            n = self._get_check_name('exclusive_hooks', app=app)
            s = "OK"
            found = []
            for i in mutually_exclusive:
                if i in self.manifest['hooks'][app]:
                    found.append(i)
            if len(found) > 1:
                t = 'error'
                s = "'%s' hooks should not be used together" % ", ".join(found)
            self._add_result(t, n, s)

        for app in self.manifest['hooks']:
            if "apparmor" in self.manifest['hooks'][app]:
                t = 'info'
                n = self._get_check_name('sdk_security_extension', app=app)
                s = "OK"
                fn = self.manifest['hooks'][app]['apparmor']
                if not fn.endswith(".apparmor"):
                    t = 'info'
                    s = '%s does not end with .apparmor (ok if not using sdk)' % fn
                self._add_result(t, n, s)

    def check_hooks_unknown(self):
        '''Check if have any unknown hooks'''
        if not self.is_click and not self.is_snap1:
            return

        #  oem snaps don't have a hooks entry
        if self.is_snap_oem:
            return

        t = 'info'
        n = self._get_check_name('unknown_hooks')
        s = 'OK'

        # Verify keys are well-formatted
        for app in self.manifest['hooks']:
            for hook in self.manifest['hooks'][app]:
                t = 'info'
                n = self._get_check_name('hooks_known', app=app, extra=hook)
                s = "OK"
                if hook not in self.known_hooks:
                    t = 'warn'
                    s = "unknown hook '%s' in %s" % (hook, app)
                self._add_result(t, n, s)

    def check_hooks_redflagged(self):
        '''Check if have any redflagged hooks'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('redflagged_hooks')
        s = 'OK'

        for app in self.manifest['hooks']:
            found = []
            t = 'info'
            n = self._get_check_name('hooks_redflag', app=app)
            s = "OK"
            manual_review = False
            for hook in self.manifest['hooks'][app]:
                if hook in self.redflagged_hooks:
                    # This check is handled elsewhere
                    if self.is_snap1 and hook == "apparmor-profile":
                        continue
                    found.append(hook)
            if len(found) > 0:
                t = 'error'
                s = "(NEEDS REVIEW) '%s' not allowed" % ", ".join(found)
                manual_review = True
            self._add_result(t, n, s, manual_review=manual_review)

    def check_external_symlinks(self):
        '''Check if symlinks in the click package go out to the system.'''
        if not self.is_click and not self.is_snap1:
            return

        if self.is_snap1 and 'type' in self.pkg_yaml and \
                self.pkg_yaml['type'] not in ['app', 'framework']:
            return

        t = 'info'
        n = self._get_check_name('external_symlinks')
        s = 'OK'

        links = find_external_symlinks(self.unpack_dir, self.pkg_files)
        if len(links) > 0:
            t = 'error'
            s = 'package contains external symlinks: %s' % ', '.join(links)
        self._add_result(t, n, s)

    def check_pkgname(self):
        '''Check click package name valid'''
        if not self.is_click:
            return
        p = self.manifest['name']
        # http://www.debian.org/doc/debian-policy/ch-controlfields.html
        t = 'info'
        n = self._get_check_name('pkgname_valid')
        s = "OK"
        if not self._verify_pkgname(p):
            t = 'error'
            s = "'%s' not properly formatted" % p
        self._add_result(t, n, s)

    def check_version(self):
        '''Check click package version is valid'''
        if not self.is_click and not self.is_snap1:
            return

        # deb-version(5)
        t = 'info'
        n = self._get_check_name('version_valid')
        s = "OK"
        # From debian_support.py
        re_valid_version = re.compile(r'^((\d+):)?'              # epoch
                                      '([A-Za-z0-9.+:~-]+?)'     # upstream
                                      '(-([A-Za-z0-9+.~]+))?$')  # debian
        if not re_valid_version.match(self.click_version):
            t = 'error'
            s = "'%s' not properly formatted" % self.click_version
        self._add_result(t, n, s)

    def check_architecture(self):
        '''Check click package architecture in DEBIAN/control is valid'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('control_architecture_valid')
        s = 'OK'
        if self.pkg_arch[0] not in self.valid_control_architectures:
            t = 'error'
            s = "not a valid architecture: %s" % self.pkg_arch[0]
        self._add_result(t, n, s)

    def check_architecture_all(self):
        '''Check if actually architecture all'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('control_architecture_valid_contents')
        s = 'OK'
        if self.pkg_arch[0] != "all":
            self._add_result(t, n, s)
            return

        # look for compiled code
        x_binaries = []
        for i in self.pkg_bin_files:
            x_binaries.append(os.path.relpath(i, self.unpack_dir))
        if len(x_binaries) > 0:
            t = 'error'
            s = "found binaries for architecture 'all': %s" % \
                ", ".join(x_binaries)
        self._add_result(t, n, s)

    def check_architecture_specified_needed(self):
        '''Check if the specified architecture is actually needed'''
        if not self.is_click and not self.is_snap1:
            return

        for arch in self.pkg_arch:
            t = 'info'
            n = self._get_check_name('architecture_specified_needed')
            s = 'OK'
            if arch == "all":
                s = "SKIPPED: architecture is 'all'"
                self._add_result(t, n, s)
                return

            if len(self.pkg_bin_files) == 0:
                t = 'warn'
                s = "Could not find compiled binaries for architecture '%s'" % \
                    arch
            self._add_result(t, n, s)

    def check_maintainer(self):
        '''Check manifest maintainer()'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('maintainer_present')
        s = 'OK'
        if 'maintainer' not in self.manifest:
            if self.is_snap1:
                s = 'Skipped optional maintainer field not specified in ' + \
                    'manifest'
            else:
                t = 'error'
                s = 'required maintainer field not specified in manifest'
            self._add_result(t, n, s)
            return
        self._add_result(t, n, s)

        # Don't perform maintainer checks for snaps. They aren't used by
        # anything.
        if self.is_snap1:
            return

        # Simple regex as used by python3-debian. If we wanted to be more
        # thorough we could use email_re from django.core.validators
        t = 'info'
        n = self._get_check_name('maintainer_format')
        s = 'OK'
        if self.manifest['maintainer'] == "":
            self._add_result('error', n, 'invalid maintainer (empty), (should be '
                                         'like "Joe Bloggs <joe.bloggs@isp.com>")',
                             'http://askubuntu.com/questions/417351/what-does-lint-maintainer-format-mean/417352')
            return
        elif not self._verify_maintainer(self.manifest['maintainer']):
            self._add_result('error', n,
                             'invalid format for maintainer: %s (should be '
                             'like "Joe Bloggs <joe.bloggs@isp.com>")' %
                             self.manifest['maintainer'],
                             'http://askubuntu.com/questions/417351/what-does-lint-maintainer-format-mean/417352')
            return
        self._add_result(t, n, s)

    def check_title(self):
        '''Check manifest title()'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('title_present')
        s = 'OK'
        if 'title' not in self.manifest:
            s = 'required title field not specified in manifest'
            self._add_result('error', n, s)
            return
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('title')
        s = 'OK'
        pkgname_base = self.click_pkgname.split('.')[-1]
        if len(self.manifest['title']) < len(pkgname_base):
            t = 'info'
            s = "'%s' may be too short" % self.manifest['title']
        self._add_result(t, n, s)

    def check_description(self):
        '''Check manifest description()'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('description_present')
        s = 'OK'
        if 'description' not in self.manifest:
            s = 'required description field not specified in manifest'
            self._add_result('error', n, s)
            return
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('description')
        s = 'OK'
        pkgname_base = self.click_pkgname.split('.')[-1]
        if len(self.manifest['description']) < len(pkgname_base):
            t = 'warn'
            if self.is_snap1 and (self.manifest['description'] == '\n' or
                                  self.manifest['description'] == ''):
                s = "manifest description is empty. Is meta/readme.md " + \
                    "formatted correctly?"
            else:
                s = "'%s' is too short" % self.manifest['description']

        self._add_result(t, n, s)

    def check_framework(self):
        '''Check manifest framework()'''
        if not self.is_click and not self.is_snap1:
            return

        # Can check with:
        # click-check-lint <path> '{"framework": {"ubuntu-sdk-15.04": {
        # "state": "obsolete", "policy_vendor": "ubuntu-core",
        # "policy_version": "16.04"}}}'
        n = self._get_check_name('framework')
        l = "http://askubuntu.com/questions/460512/what-framework-should-i-use-in-my-manifest-file"
        framework_overrides = self.overrides.get('framework', {})
        frameworks = Frameworks(overrides=framework_overrides)

        if not self.is_snap1 and ',' in self.manifest['framework']:
            # click doesn't support multiple frameworks yet
            t = 'error'
            s = 'ERROR: multiple frameworks found in click manifest'
            self._add_result(t, n, s)
            return

        for framework in self.manifest['framework'].split(','):
            framework = framework.strip()
            if framework in frameworks.OBSOLETE_FRAMEWORKS:
                t = 'error'
                s = "'%s' is obsolete. Please use a newer framework" % \
                    framework
                self._add_result(t, n, s, l)
                return
            elif framework in frameworks.DEPRECATED_FRAMEWORKS:
                t = 'warn'
                s = "'%s' is deprecated. Please use a newer framework" % \
                    framework
                self._add_result(t, n, s, l)
                return
            elif framework in frameworks.AVAILABLE_FRAMEWORKS:
                t = 'info'
                s = 'OK'
                self._add_result(t, n, s)
                return
            else:
                # None of the above checks triggered, this is an unknown framework
                t = 'error'
                s = "'%s' is not a supported framework" % \
                    framework
                self._add_result(t, n, s, l)

    def check_click_local_extensions(self):
        '''Report any click local extensions'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('click_local_extensions')
        s = 'OK'
        found = []
        for k in sorted(self.manifest):
            if k.startswith('x-'):
                found.append(k)

        # Some local extensions are ok for core apps, so just skip them
        if self.is_core_app:
            for i in ['x-source', 'x-test']:
                if i in found:
                    found.remove(i)

        if len(found) > 0:
            t = 'warn'
            plural = ""
            if len(found) > 1:
                plural = "s"
            s = 'found unofficial extension%s: %s' % (plural, ', '.join(found))
        self._add_result(t, n, s)

    def check_vcs(self):
        '''Check for VCS files in the package'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('vcs_files')
        s = 'OK'
        found = []
        for d in self.vcs_files:
            entries = glob.glob("%s/%s" % (self.unpack_dir, d))
            if len(entries) > 0:
                for i in entries:
                    found.append(os.path.relpath(i, self.unpack_dir))
        if len(found) > 0:
            t = 'warn'
            s = 'found VCS files in package: %s' % ", ".join(found)
        self._add_result(t, n, s)

    def check_click_in_package(self):
        '''Check for *.click files in the toplevel click package'''
        if not self.is_click:
            return

        t = 'info'
        n = self._get_check_name('click_files')
        s = 'OK'
        found = []
        entries = glob.glob("%s/*.click" % self.unpack_dir)
        if len(entries) > 0:
            for i in entries:
                found.append(os.path.relpath(i, self.unpack_dir))
        if len(found) > 0:
            t = 'warn'
            s = 'found click packages in toplevel dir: %s' % ", ".join(found)
        self._add_result(t, n, s)

    def check_dot_click(self):
        '''Check for .click directory in the toplevel click package'''
        if not self.is_click and not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('dot_click')
        s = 'OK'

        path = os.path.join(self.unpack_dir, '.click')
        if os.path.exists(path):
            t = 'error'
            s = 'found .click in toplevel dir'
        self._add_result(t, n, s)

    def check_contents_for_hardcoded_paths(self):
        '''Check for known hardcoded paths.'''
        if not self.is_click and not self.is_snap1:
            return

        PATH_BLACKLIST = ["/opt/click.ubuntu.com/"]
        t = 'info'
        n = self._get_check_name('hardcoded_paths')
        s = 'OK'
        for dirpath, dirnames, filenames in os.walk(self.unpack_dir):
            for filename in filenames:
                full_fn = os.path.join(dirpath, filename)
                (rc, out) = cmd(['file', '-b', full_fn])
                if 'text' not in out:
                    continue
                try:
                    lines = open_file_read(full_fn).readlines()
                    for bad_path in PATH_BLACKLIST:
                        if list(filter(lambda line: bad_path in line, lines)):
                            t = 'error'
                            s = "Hardcoded path '%s' found in '%s'." % (
                                bad_path, full_fn)
                except FileNotFoundError:
                    pass
                except UnicodeDecodeError:
                    pass
        self._add_result(t, n, s)

    def _verify_architecture(self, my_dict, test_str):
        t = 'info'
        n = self._get_check_name('%s_architecture_valid' % test_str)
        s = 'OK'
        if 'architecture' not in my_dict and 'architectures' not in my_dict:
            s = 'OK (architecture not specified)'
            self._add_result(t, n, s)
            return

        key = 'architecture'
        if self.is_snap1 and 'architectures' in my_dict:
            # new yaml allows for 'architecture' and 'architectures'
            key = 'architectures'

        archs_list = list(self.valid_control_architectures)
        archs_list.remove("multi")

        if self.is_snap1 and key == 'architectures' and \
           isinstance(my_dict[key], str):
            # new yaml uses 'architectures' that must be a list
            t = 'error'
            s = "not a valid architecture: %s (not a list)" % my_dict[key]
        elif isinstance(my_dict[key], str) and my_dict[key] not in archs_list:
            t = 'error'
            s = "not a valid architecture: %s" % my_dict[key]
        elif isinstance(my_dict[key], list):
            if not self.is_snap1:
                archs_list.remove("all")
            bad_archs = []
            for a in my_dict[key]:
                if a not in archs_list:
                    bad_archs.append(a)
                if len(bad_archs) > 0:
                    t = 'error'
                    s = "not valid multi architecture: %s" % \
                        ",".join(bad_archs)
        self._add_result(t, n, s)

    def check_manifest_architecture(self):
        '''Check package architecture in manifest is valid'''
        if not self.is_click and not self.is_snap1:
            return

        self._verify_architecture(self.manifest, "manifest")

    def _verify_icon(self, my_dict, test_str):
        t = 'info'
        n = self._get_check_name('%s_icon_present' % test_str)
        s = 'OK'
        if 'icon' not in my_dict:
            s = 'Skipped, optional icon not present'
            self._add_result(t, n, s)
            return
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('%s_icon_empty' % test_str)
        s = 'OK'
        if len(my_dict['icon']) == 0:
            t = 'error'
            s = "icon entry is empty"
            return
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('%s_icon_absolute_path' % test_str)
        s = 'OK'
        if my_dict['icon'].startswith('/'):
            t = 'error'
            s = "icon entry '%s' should not specify absolute path" % \
                my_dict['icon']
        self._add_result(t, n, s)

    def check_icon(self):
        '''Check icon()'''
        if not self.is_click and not self.is_snap1:
            return

        self._verify_icon(self.manifest, "manifest")

    def check_snappy_name(self):
        '''Check package name'''
        if self.is_click or not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('snappy_name_valid')
        s = 'OK'
        if 'name' not in self.pkg_yaml:
            t = 'error'
            s = "could not find 'name' in yaml"
        elif not self._verify_pkgname(self.pkg_yaml['name']):
            t = 'error'
            s = "malformed 'name': '%s'" % self.pkg_yaml['name']
        self._add_result(t, n, s)

    def check_snappy_version(self):
        '''Check package version'''
        if self.is_click or not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('snappy_version_valid')
        s = 'OK'
        if 'version' not in self.pkg_yaml:
            t = 'error'
            s = "could not find 'version' in yaml"
        elif not self._verify_pkgversion(self.pkg_yaml['version']):
            t = 'error'
            s = "malformed 'version': '%s'" % self.pkg_yaml['version']
        self._add_result(t, n, s)

    def check_snappy_type(self):
        '''Check type'''
        if self.is_click or not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('snappy_type_valid')
        s = 'OK'
        if 'type' not in self.pkg_yaml:
            s = 'OK (skip missing)'
        elif self.pkg_yaml['type'] not in self.snappy_valid_types:
            t = 'error'
            s = "unknown 'type': '%s'" % self.pkg_yaml['type']
        self._add_result(t, n, s)

    def check_snappy_type_redflagged(self):
        '''Check if snappy type is redflagged'''
        if self.is_click or not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('snappy_type_redflag')
        s = "OK"
        l = None
        manual_review = False
        if 'type' not in self.pkg_yaml:
            s = 'OK (skip missing)'
        elif self.pkg_yaml['type'] in self.snappy_redflagged_types:
            t = 'error'
            s = "(NEEDS REVIEW) type '%s' not allowed" % self.pkg_yaml['type']
            manual_review = True
            if self.pkg_yaml['type'] == "framework":
                l = "https://developer.ubuntu.com/en/snappy/guides/frameworks/"
        self._add_result(t, n, s, link=l, manual_review=manual_review)

    def check_snappy_icon(self):
        '''Check icon()'''
        if self.is_click or not self.is_snap1:
            return

        self._verify_icon(self.pkg_yaml, "package_yaml")

    def check_snappy_architecture(self):
        '''Check package architecture in package.yaml is valid'''
        if self.is_click or not self.is_snap1:
            return

        self._verify_architecture(self.pkg_yaml, "package yaml")

        t = 'info'
        n = self._get_check_name('snappy_architecture_deprecated')
        s = 'OK'
        l = None
        if 'architecture' in self.pkg_yaml:
            t = 'warn'
            s = "Found deprecated 'architecture' field in " + \
                "meta/package.yaml. Use 'architectures' instead"
            l = 'https://developer.ubuntu.com/en/snappy/guides/package-metadata/'
        self._add_result(t, n, s, link=l)

    def check_snappy_unknown_entries(self):
        '''Check for any unknown fields'''
        if self.is_click or not self.is_snap1:
            return

        t = 'info'
        n = self._get_check_name('snappy_unknown')
        s = 'OK'
        unknown = []
        for f in self.pkg_yaml:
            if f not in self.snappy_required + self.snappy_optional:
                unknown.append(f)
        if len(unknown) > 0:
            t = 'warn'
            s = "unknown entries in package.yaml: '%s'" % \
                (",".join(sorted(unknown)))
            obsoleted = ['maintainer', 'ports']
            tmp = list(set(unknown) & set(obsoleted))
            if len(tmp) > 0:
                t = 'error'
                s += " (%s obsoleted)" % ",".join(tmp)
        self._add_result(t, n, s)

    def _extract_readme_md(self):
        '''Extract meta/readme.md'''
        contents = None
        readme = os.path.join(self.unpack_dir, "meta/readme.md")
        if os.path.exists(readme):
            fh = open_file_read(readme)
            contents = fh.read()
        return contents

    def check_snappy_readme_md(self):
        '''Check snappy readme.md'''
        if self.is_click or not self.is_snap1:
            return

        contents = self._extract_readme_md()

        t = 'info'
        n = self._get_check_name('snappy_readme.md')
        s = 'OK'
        if contents is None:
            t = 'error'
            s = 'meta/readme.md does not exist'
            self._add_result(t, n, s)
            return
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('snappy_readme.md_length')
        s = 'OK'
        pkgname_base = self.pkg_yaml['name'].split('.')[0]
        if len(contents) < len(pkgname_base):
            t = 'warn'
            s = "meta/readme.md is too short"
        self._add_result(t, n, s)

    def check_snappy_config(self):
        '''Check snappy config'''
        if self.is_click or not self.is_snap1:
            return

        fn = os.path.join(self.unpack_dir, 'meta/hooks/config')
        if fn not in self.pkg_files:
            return

        t = 'info'
        n = self._get_check_name('snappy_config_hook_executable')
        s = 'OK'
        if not self._check_innerpath_executable(fn):
            t = 'error'
            s = 'meta/hooks/config is not executable'
        self._add_result(t, n, s)

    def check_snappy_services_and_binaries(self):
        '''Services and binaries should not overlap'''
        if self.is_click or not self.is_snap1:
            return
        for exe_t in ['binaries', 'services']:
            if exe_t not in self.pkg_yaml:
                break
            if exe_t == 'binaries':
                other_exe_t = 'services'
            else:
                other_exe_t = 'binaries'

            if other_exe_t not in self.pkg_yaml:
                break

            for a in self.pkg_yaml[exe_t]:
                if 'name' not in a:
                    continue
                # handle 'exec' in binaries
                app = os.path.basename(a['name'])

                t = 'info'
                n = self._get_check_name(
                    'snappy_in_%s' % other_exe_t, app=app)
                s = 'OK'
                for other_a in self.pkg_yaml[other_exe_t]:
                    other_app = os.path.basename(other_a['name'])
                    if app == other_app:
                        t = 'error'
                        s = "'%s' in both 'services' and 'binaries'" % app
                        break
                self._add_result(t, n, s)

    def check_snappy_hashes(self):
        '''Check snappy hashes.yaml'''
        if self.is_click or not self.is_snap1:
            return

        def _check_allowed_perms(mode, allowed):
            '''Check that mode only uses allowed perms'''
            for p in mode[1:]:
                if p not in allowed:
                    return False
            return True

        try:
            hashes_yaml = yaml.safe_load(self._extract_hashes_yaml())
        except Exception:
            error("Could not load hashes.yaml. Is it properly formatted?")

        if 'archive-sha512' not in hashes_yaml:
            t = 'error'
            n = self._get_check_name('hashes_archive-sha512_present')
            s = "'archive-sha512' not found in hashes.yaml"
            self._add_result(t, n, s)
            return

        # verify the ar file
        t = 'info'
        n = self._get_check_name('hashes_archive-sha512_valid')
        s = 'OK'
        fn = self._path_join(self.raw_unpack_dir, 'data.tar.gz')
        sum = self._get_sha512sum(fn)
        if hashes_yaml['archive-sha512'] != sum:
            t = 'error'
            s = "hash mismatch: '%s' != '%s'" % (hashes_yaml['archive-sha512'],
                                                 sum)
            self._add_result(t, n, s)
            return
        self._add_result(t, n, s)

        if 'files' not in hashes_yaml:
            t = 'error'
            n = self._get_check_name('hashes_files_present')
            s = "'files' not found in hashes.yaml"
            self._add_result(t, n, s)
            return

        # verify the individual files
        errors = []
        badsums = []
        hash_files = set([])  # used to check with extra files
        for entry in hashes_yaml['files']:
            if 'name' not in entry:
                errors.append("'name' not found for entry '%s'" % entry)
                continue
            else:
                hash_files.add(entry['name'])

            if 'mode' not in entry:
                errors.append("'mode' not found for entry '%s'" %
                              entry['name'])
                continue
            elif len(entry['mode']) != 10:
                errors.append("malformed mode '%s' for entry '%s'" %
                              (entry['mode'], entry['name']))
                continue
            elif entry['mode'].startswith('d'):
                if not _check_allowed_perms(entry['mode'],
                                            ['r', 'w', 'x', '-']):
                    errors.append("unusual mode '%s' for entry '%s'" %
                                  (entry['mode'], entry['name']))
                # world write shouldn't be allowed
                if entry['mode'][-2] != '-':
                    errors.append("'%s' is world-writable" % entry['name'])
                continue
            elif entry['mode'].startswith('l'):
                if entry['mode'] != "lrwxrwxrwx":
                    errors.append("unusual mode '%s' for entry '%s'" %
                                  (entry['mode'], entry['name']))
                continue
            elif not entry['mode'].startswith('f'):
                # files and symlinks are ok, everything else is not
                errors.append("illegal file mode '%s': '%s' for '%s'" %
                              (entry['mode'][0], entry['mode'], entry['name']))
                continue
            elif 'size' not in entry:
                errors.append("'size' not found for entry: %s" % entry['name'])
                continue
            elif 'sha512' not in entry:
                errors.append("'sha512' not found for entry: %s" %
                              entry['name'])
                continue

            fn = self._path_join(self.unpack_dir, entry['name'])

            # quick verify the size, if it is wrong, we don't have to do the
            # sha512sum
            statinfo = self._extract_statinfo(fn)
            # file is missing
            if statinfo is None:
                errors.append("'%s' does not exist" % entry['name'])
                continue
            if entry['size'] != statinfo.st_size:
                errors.append("size %d != %d for '%s'" % (entry['size'],
                                                          statinfo.st_size,
                                                          entry['name']))
                continue

            # check for weird perms
            if not _check_allowed_perms(entry['mode'], ['r', 'w', 'x', '-']):
                errors.append("unusual mode '%s' for entry '%s'" %
                              (entry['mode'], entry['name']))
                continue
            # world write shouldn't be allowed
            if entry['mode'][-2] != '-':
                errors.append("mode '%s' for '%s' is world-writable" %
                              (entry['mode'], entry['name']))
                continue
            # verify perms match what is in hashes.yaml
            filemode = stat.filemode(statinfo.st_mode)[1:]
            if entry['mode'][1:] != filemode:
                errors.append("mode '%s' != '%s' for '%s'" %
                              (entry['mode'][1:], filemode, entry['name']))
                continue

            # ok, now all the cheap tests are done so we can check if we have a
            # valid sha512sum
            sum = self._get_sha512sum(fn)
            if entry['sha512'] != sum:
                badsums.append("'%s' != '%s' for '%s'" % (entry['sha512'], sum,
                                                          entry['name']))

        t = 'info'
        n = self._get_check_name('sha512sums')
        s = 'OK'
        if len(badsums) > 0:
            t = 'error'
            s = 'found bad checksums: %s' % ", ".join(badsums)
        self._add_result(t, n, s)

        t = 'info'
        n = self._get_check_name('file_mode')
        s = 'OK'
        if len(errors) > 0:
            t = 'error'
            s = 'found errors in hashes.yaml: %s' % ", ".join(errors)
        self._add_result(t, n, s)

        # Now check for extra files
        t = 'info'
        n = self._get_check_name('hashes_extra_files')
        s = 'OK'
        self._add_result(t, n, s)
        extra = []
        click_compat_files = set(["DEBIAN/hashes.yaml",
                                  "DEBIAN/control",
                                  "DEBIAN/preinst",
                                  "DEBIAN/manifest"
                                  ])
        for f in self.pkg_files:
            fn = os.path.relpath(f, self.unpack_dir)
            if fn not in hash_files and fn not in click_compat_files:
                extra.append(fn)

        if len(extra) > 0:
            t = 'error'
            s = 'found extra files not listed in hashes.yaml: %s' % \
                ", ".join(extra)
        self._add_result(t, n, s)

    def check_snappy_frameworks(self):
        '''TODO'''
        return False
