#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Copyright © 2015 Collabora Ltd.
#
# SPDX-License-Identifier: MPL-2.0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# build-snapshot — Build snapshots of a package.
#
# Assumptions:
# * The current working directory is the root of a git repository
# * The current branch has upstream sources and a debian/ directory
# * Upstream tags look like v1.2.3
# * Debian packaging tags look like debian/1.2.3-4 (or vendor/1.2.3-4vendor5)
# * mktemp uses a location with plenty of space
# * The upstream project uses Autotools
# * "NOCONFIGURE=1 ./autogen.sh" and "./configure" lead to a reasonable build
# * Parallel "make" and "make distcheck" work

import argparse
import fnmatch
import glob
import os
import shlex
import subprocess
import sys

# python3-debian or python-debian is available in Debian, Fedora, etc.
from debian.debian_support import Version

class SnapshotBuilder:
    def __init__(self, args):
        self.args = args
        self.srcdir = '.'

    def to_shell(self, cmd, shell=False):
        """cmd is the first argument of subprocess.call:
        a sequence of argv elements if shell is false, or a single
        shell one-liner if shell is true. Return an equivalent
        shell one-liner, suitable for running via sh -c or ssh.
        """
        if shell:
            return cmd
        else:
            return ' '.join((shlex.quote(x) for x in cmd))

    def do(self, cmd, shell=False):
        """Log cmd, and execute it on self.args.builder if not in
        --dry-run mode. Raise CalledProcessException if it fails.

        If shell is false, cmd is a sequence of argv elements;
        if true, cmd is a single string containing a shell one-liner.
        """
        if self.args.builder == 'localhost':
            self.do_locally(cmd, shell=shell)
        else:
            # If we’re running on a TTY, run SSH in interactive mode, so that
            # the TERM variable is set correctly on the remote host and hence
            # we end up with compiler coloured output.
            interactive = '-t' if sys.stderr.isatty() else '-T'
            self.do_locally(['ssh', interactive, self.args.builder,
                self.to_shell(cmd, shell)], shell=False)

    def do_locally(self, cmd, shell=False):
        """Same as do(), but run cmd locally.
        """
        if shell:
            cmd = ['sh', '-c', cmd]

        print('# {}'.format(cmd))

        if self.args.really:
            subprocess.check_call(cmd)

    def do_in_srcdir(self, cmd, shell=False):
        """Same as do(), but cd to srcdir first.
        """
        cmd = self.to_shell(cmd, shell)
        self.do('cd {} && {}'.format(shlex.quote(self.srcdir), cmd),
                shell=True)

    def capture_locally(self, cmd, shell=False):
        """Same as do_locally(), but capture and return the stdout from cmd.
        """
        if shell:
            cmd = ['sh', '-c', cmd]

        print('# {}'.format(cmd))

        if self.args.really:
            return subprocess.check_output(cmd, universal_newlines=True)
        else:
            return '«output of {}»'.format(cmd)

    def capture(self, cmd, shell=False):
        """Same as do(), but capture and return the stdout from cmd.
        """
        if self.args.builder == 'localhost':
            return self.capture_locally(cmd, shell=shell)
        else:
            cmd = self.to_shell(cmd, shell)
            return self.capture_locally(['ssh', self.args.builder, cmd])

    def capture_in_srcdir(self, cmd, shell=False):
        """Same as do_in_srcdir(), but capture and return the stdout from cmd.
        """
        cmd = self.to_shell(cmd, shell)
        return self.capture('cd {} && {}'.format(shlex.quote(self.srcdir), cmd),
                shell=True)

    def parse_description(self, description):
        """Parse `git describe --long` output.

        Return the name of the tag we used as a reference point,
        the Version that it represents, the number of commits since
        that tag to get to where we are now, and the short commit hash
        of where we are now.
        """
        # (VVV, [WW,] NN, gCCCCCCC)
        description = description.split('-')
        assert len(description) >= 3, description
        # VVV[-WW]
        tag = '-'.join(description[:-2])

        if '/' in tag:
            # chop off debian/ resulting in VVV[-WW]-NN-gCCCCCCC
            tagged_version = tag.split('/', 1)[1]
        elif tag.startswith('v'):
            tagged_version = tag[1:]
        else:
            tagged_version = tag

        # undo gbp version mangling: 1%1.2_beta3 -> 1:1.2~beta3
        tagged_version = tagged_version.replace('%', ':').replace('_', '~')
        # NN
        counter = description[-2]
        # CCCCCCC
        commit = description[-1].lstrip('g')
        return tag, Version(tagged_version), int(counter), commit

    def count_all_commits(self):
        """Fallback mode if we can't find a tag: count all the commits since
        the beginning of history.

        Return the number of commits that have ever happened, and the
        short commit hash of the current commit. This is analogous to
        the last two return values from parse_description(), if we
        imagine that the tag found by `git describe` was just before the
        first commit to this repository (which of course can't happen).
        """
        counter = int(self.capture_locally('git rev-list HEAD | wc -l',
            shell=True).strip())
        commit = self.capture_locally(['git', 'show', '--pretty=format:%h',
            '-s']).strip()
        return counter, commit

    def do_merges(self):
        for r in self.args.remotes:
            name, uri = r.split('=', 1)
            try:
                self.do_in_srcdir(['git', 'remote', 'remove', name])
            except subprocess.CalledProcessError:
                pass
            self.do_in_srcdir(['git', 'remote', 'add', name, uri])
            self.do_in_srcdir(['git', 'remote', 'update', '--prune', name])

        for m in self.args.merges:
            self.do_in_srcdir(['git', 'merge',
                    '-s', self.args.merge_strategy,
                    '-m', 'Automatic merge for snapshot',
                ] + ['-X{}'.format(x) for x in self.args.merge_options] + [
                    m,
                ])

    def main(self):
        """Main function: build the snapshot.
        """
        tmpdir = self.capture(['mktemp', '-d', '--tmpdir',
            'build-snapshot.XXXXXXXXXX']).strip('\n')
        self.srcdir = tmpdir + '/s'

        if self.args.builder == 'localhost':
            self.do_locally(['rsync', '-az', '--delete', './',
                self.srcdir + '/'])
        else:
            self.do_locally(['rsync', '-az', '--delete', './',
                self.args.builder + ':' + self.srcdir + '/'])

        self.do_merges()

        # Check build-dependencies first, so we don't waste time running
        # autoreconf if it's going to fail.
        if not self.args.source_only:
            self.do_in_srcdir(['dpkg-checkbuilddeps'])

        source_package = self.capture_in_srcdir(['dpkg-parsechangelog',
            '-SSource']).strip('\n')

        changelog_version = self.capture_in_srcdir([
            'dpkg-parsechangelog', '-SVersion', '-n1']).strip('\n')
        changelog_version = Version(changelog_version)
        is_native = (changelog_version.debian_revision is None)

        if os.path.exists('debian/source/format'):
            source_format = open('debian/source/format').read().strip()
        else:
            source_format = '1.0'

        assert source_format in ('3.0 (quilt)', '3.0 (native)', '1.0'), \
                ('debian/source/format unsupported: {}'.format(source_format))

        if source_format == '3.0 (native)':
            assert is_native
        elif source_format == '3.0 (quilt)':
            assert not is_native

        if self.args.upstream and not is_native:
            # vVVV[-WW]-NN-gCCCCCCC
            try:
                raw_description = self.capture_locally(['git', 'describe',
                    '--long', '--tags', '--match=v*']).strip()
            except subprocess.CalledProcessError:
                tag = None
                tagged_version = None
                counter, commit = self.count_all_commits()
                reference_point = 'beginning of history'
            else:
                tag, tagged_version, counter, commit = self.parse_description(
                        raw_description)
                reference_point = tag

            if counter == 0:
                # We are exactly at an upstream tag. Presumably what we
                # intend to be doing here is to release it.
                doing_upstream_release = True
            elif tagged_version is None:
                # There are no tags, so we are certainly going to have to
                # do an upstream release before we can do a downstream one.
                doing_upstream_release = True
            else:
                # We are somewhere after the upstream tag. If the only
                # differences are within debian/ then we're doing a
                # "packaging" snapshot, else we're doing an "upstream"
                # snapshot
                doing_upstream_release = False

                diffstat = self.capture_locally(['git', 'diff',
                    '--name-only', '--no-renames', '-z', tag + '..HEAD'])
                for line in diffstat.split('\0'):
                    if line and not line.startswith('debian/'):
                        doing_upstream_release = True
        else:
            doing_upstream_release = False

        if is_native or not doing_upstream_release:
            # Find the most recent Debian version that has a tag
            offset = 0
            while offset < 10:
                tagged_version = self.capture_in_srcdir(['dpkg-parsechangelog',
                    '-SVersion', '-o{}'.format(offset), '-n1']).strip('\n')
                # ~ and : are necessary syntax elements in some dpkg
                # version numbers but are not valid in git tags;
                # convert them in the same way as gbp,
                # 1:1.2~beta3 -> 1%1.2_beta3
                mangled_version = tagged_version.replace(':', '%')
                mangled_version = mangled_version.replace('~', '_')
                try:
                    raw_description = self.capture_locally(['git', 'describe',
                        '--tags', '--match=*/' + mangled_version,
                        '--long']).strip()
                except subprocess.CalledProcessError:
                    # assume there was no such tag, and try the next
                    offset += 1
                    continue
                else:
                    _, tagged_version, counter, commit = self.parse_description(
                            raw_description)
                    reference_point = str(tagged_version)
                    break
            else:
                # No recent Debian version has a tag, fall back to counting
                # every commit we've ever done (as though there was a tag
                # just before the beginning of history)
                tagged_version = None
                counter, commit = self.count_all_commits()
                reference_point = 'beginning of history'

        if tagged_version is None:
            # Nothing has been tagged yet, and the changelog says something
            # like "foo (vvv[-ww]) UNRELEASED; urgency=low". Choose a
            # version that is not only less than vvv[-ww], but also less
            # than what would appear if we did have a tag for version
            # VVV[-WW] < vvv available, by appending '~' (which sorts
            # before the empty string), twice (so it sorts before '~').
            snapshot_version = Version(changelog_version)

            if doing_upstream_release or is_native:
                snapshot_version.upstream_version += '~~'
            else:
                snapshot_version.debian_version += '~~'
        else:
            # VVV[-WW]
            snapshot_version = Version(tagged_version)

        if snapshot_version.epoch is None:
            snapshot_version.epoch = changelog_version.epoch

        if counter == 0:
            # we are exactly at a tag: use it
            if changelog_version.upstream_version == snapshot_version.upstream_version:
                # we are exactly at an upstream tag and the changelog
                # describes a corresponding Debian revision: go with that
                snapshot_version = changelog_version
        elif doing_upstream_release or is_native:
            # VVV+NN+gCCCCCCC-0~snapshot
            snapshot_version.upstream_version = (
                    '{}+{}+g{}'.format(snapshot_version.upstream_version,
                        counter, commit))
            if not is_native:
                # we need some sort of Debian revision number, and the one
                # in the changelog is meaningless because we've bumped the
                # upstream version
                snapshot_version.debian_revision = '0~snapshot'
        else:
            # VVV-WW+NN+gCCCCCCC
            snapshot_version.debian_revision = (
                    '{}+{}+g{}'.format(snapshot_version.debian_revision,
                        counter, commit))

        if (tagged_version is not None and
                changelog_version.upstream_version > Version(tagged_version)):
            # git says the latest tag is foo_1.2 but debian/changelog says
            # we are foo_2.0-1, or something. This probably means someone
            # is preparing to make a release with new API/ABI.
            # Make the snapshot version "slightly less than" the version
            # being prepared, for example 2.0~1.2+3+g567890a instead of
            # 1.2+3+g567890a, so that it satisfies backport-friendly
            # dependencies like foo (>= 2.0~).
            # Conveniently, 2.0~1.2+... is less than 2.0~beta1, so
            # genuine prereleases will normally sort higher than snapshots.
            if is_native or doing_upstream_release:
                snapshot_version.upstream_version = '{}~{}'.format(
                        changelog_version.upstream_version,
                        snapshot_version.upstream_version)
            else:
                # We have to leave the upstream version as it is, otherwise
                # we'd have the wrong orig tarball; make the Debian revision
                # small instead, like 2.0-0~1.2+0vendor1+3+g567890a
                # (which we want to be less than 2.0-0vendor1)
                snapshot_version.debian_revision = '0~{}+{}'.format(
                        snapshot_version.upstream_version,
                        snapshot_version.debian_revision).replace('-', '+')
                snapshot_version.upstream_version = changelog_version.upstream_version

        if '/' in str(snapshot_version):
            raise AssertionError('Bad version {!r}'.format(snapshot_version))
        if '/' in source_package:
            raise AssertionError('Bad package name {!r}'.format(source_package))

        path = self.capture('echo $PATH', shell=True).strip('\n')
        if ':/usr/lib/ccache:' not in ':{}:'.format(path):
            path = '/usr/lib/ccache:{}'.format(path)

        # Assume that packages with /configure.ac are Autotools.
        # We probe for configure.ac instead of testing for is_native,
        # because we could conceivably have native packages that use
        # Autotools and we want to exercise distcheck for those;
        # we don't do this unconditionally because we have some native
        # packages that don't use Autotools.
        if (self.args.upstream and os.path.exists('configure.ac') and
            not self.args.source_only and
            # If we want a source package, we must at least `make dist`.
            # If we want to do build-time tests, we presumably want to
            # also verify that it distchecks. However, if we just want
            # binary packages as fast as possible (--no-check),
            # skip it.
            (self.args.source or self.args.check)):
            if os.path.exists('.git'):
                self.do_in_srcdir(['git', 'status', '-u'])
                self.do_in_srcdir(['git', 'clean', '-fxd'])

            self.do_in_srcdir(['env', 'NOCONFIGURE=1', './autogen.sh'])
            self.do_in_srcdir(['./configure'])

            if self.args.check:
                self.do_in_srcdir(['env', 'PATH={}'.format(path), 'make',
                    '-j{}'.format(self.args.jobs), 'check', 'VERBOSE=1',
                    'CFLAGS=-fdiagnostics-color=auto'])
                self.do_in_srcdir(['env', 'PATH={}'.format(path), 'make',
                    '-j{}'.format(self.args.jobs), 'distcheck', 'VERBOSE=1',
                    'CFLAGS=-fdiagnostics-color=auto'])
            else:
                self.do_in_srcdir(['env', 'PATH={}'.format(path), 'make',
                    '-j{}'.format(self.args.jobs), 'VERBOSE=1',
                    'CFLAGS=-fdiagnostics-color=auto'])
                self.do_in_srcdir(['env', 'PATH={}'.format(path), 'make',
                    '-j{}'.format(self.args.jobs), 'dist', 'VERBOSE=1',
                    'CFLAGS=-fdiagnostics-color=auto'])

        if self.args.source and not is_native:
            if doing_upstream_release:
                if self.args.source_only:
                    raise AssertionError('Cannot "make distcheck" for an '
                        'upstream release in a source-only build')

                # assume we already built the tarball via make distcheck
                assert os.path.exists('configure.ac')
                assert self.args.upstream

                tarball = self.capture_in_srcdir('ls *.tar.*', shell=True)
                tarball = tarball.strip()
                # assume there should only be one
                assert '\n' not in tarball, tarball

                for ext in '.tar.xz', '.tar.gz':
                    if tarball.endswith(ext):
                        break
                else:
                    raise AssertionError(
                            'Unsupported tarball extension: {!r}'.format(
                                tarball))

                self.do_in_srcdir('mv {} ../{}_{}.orig{}'.format(
                    tarball, shlex.quote(source_package),
                    shlex.quote(snapshot_version.upstream_version),
                    ext),
                    shell=True)
            else:
                tarball_format = '{}_{}.orig.tar.*'.format(
                    source_package, changelog_version.upstream_version)
                tarballs = glob.glob('{}/{}'.format(self.args.orig_dir,
                    tarball_format))
                got_tarballs = False

                if tarballs:
                    if self.args.builder == 'localhost':
                        dest = '{}/'.format(tmpdir)
                    else:
                        dest = '{}:{}/'.format(self.args.builder, tmpdir)

                    for t in tarballs:
                        self.do_locally(['rsync', '-P', '--copy-links', t,
                            dest])

                    got_tarballs = True

                if not got_tarballs:
                    try:
                        self.do_in_srcdir(['git', 'branch', 'pristine-tar',
                            'origin/pristine-tar'])
                    except subprocess.CalledProcessError:
                        # assume branch already exists
                        pass
                    try:
                        for line in self.capture_in_srcdir(['pristine-tar',
                                'list']).splitlines():
                            if fnmatch.fnmatch(line, tarball_format):
                                self.do_in_srcdir(['pristine-tar', 'checkout',
                                    line])
                                self.do_in_srcdir(['mv', line, '..'])
                                got_tarballs = True
                                break
                    except subprocess.CalledProcessError:
                        pass

                if not got_tarballs:
                    self.do('cd {} && apt-get --download-only source {}'.format(
                        shlex.quote(tmpdir),
                        shlex.quote(source_package),
                        ), shell=True)

        if os.path.exists('.git'):
            self.do_in_srcdir(['git', 'status', '-u'])
            self.do_in_srcdir(['git', 'clean', '-fxd'])

        # If the version in the changelog is not already what we want,
        # make it so
        if changelog_version != snapshot_version:
            self.do_in_srcdir(['env',
                'DEBFULLNAME=Snapshot',
                'DEBEMAIL=snapshot@localhost',
                'dch', '-v{}'.format(snapshot_version), '-b',
                'snapshot: commit {} ({} commits after {})'.format(
                    commit, counter, reference_point)])

        if self.args.source_only:
            args = ['-S', '-d']
        elif self.args.source:
            args = []
        else:
            args = ['-b']

        deb_build_options = set(self.args.deb_build_options)

        if not self.args.check:
            deb_build_options.add('nocheck')

        self.do_in_srcdir(['env', 'PATH={}'.format(path),
            'DEB_BUILD_OPTIONS={}'.format(' '.join(deb_build_options)),
            'dpkg-buildpackage', '-J{}'.format(self.args.jobs),
            '-us', '-uc', '-i', '-I'] + args)
        if os.path.exists('.git'):
            self.do_in_srcdir(['git', 'status', '-u'])

        if self.args.download is not None:
            if self.args.builder != 'localhost':
                self.do_locally(['rsync', '-zvP',
                    '{}:{}/*'.format(self.args.builder, tmpdir),
                    '{}/'.format(self.args.download)])
            else:
                self.do_locally(['rsync', '-zvP'] +
                    list(glob.glob('{}/*'.format(tmpdir))) +
                    ['{}/'.format(self.args.download)])

        if ((self.args.install or self.args.install_all) and
                not self.args.source_only):
            if (self.args.builder == 'localhost' and
                    not self.args.force_local_install):
                raise SystemExit('Refusing to install built packages locally '
                        'without --force-local-install')

            packages = self.capture_in_srcdir(
                    'ls {}/*.deb'.format(shlex.quote(tmpdir)),
                    shell=True)
            install = (['sudo', 'apt', 'install'] +
                    list(packages.strip('\n').splitlines()))

            if self.args.install_all:
                install.append('--allow-downgrades')
                install.append('-y')
            else:
                install.append('--only-upgrade')

            self.do_in_srcdir(install)

            if os.path.exists('debian/tests/control'):
                # sadt might do a compile-test, so use ccache here too
                self.do_in_srcdir(['env', 'PATH={}'.format(path), 'sadt',
                    '--verbose', '--built-source-tree'])

        if not self.args.keep:
            self.do(['rm', '-rf', shlex.quote(tmpdir)])

def main():
    parser = argparse.ArgumentParser(description='Make a snapshot of a package')

    parser.add_argument('--source', '-s',
            help='Build a source package (default: in-tree binary-only build)',
            action='store_true', default=False)
    parser.add_argument('--source-only', '-S',
            help='Build a source package only',
            action='store_true', default=False, dest='source_only')

    parser.add_argument('--dry-run', '-n',
            help='Do not really run build commands (default: really build)',
            action='store_false', default=True, dest='really')
    parser.add_argument('--no-check', '--nocheck',
            help='do not run build-time tests',
            action='store_false', dest='check', default=True)
    parser.add_argument('--build-option', '-O',
            help='append this to DEB_BUILD_OPTIONS (noopt, nocheck etc.)',
            action='append', dest='deb_build_options', default=[])

    parser.add_argument('--packaging', '--packaging-only', '-p',
            help='assume that we are only packaging this package',
            action='store_false', dest='upstream', default=True)
    parser.add_argument('--upstream', '-u',
            help='assume that we are the upstream for this package (default)',
            action='store_true', default=True)
    parser.add_argument('--orig-dir',
            help='with -p, look for orig.tar.gz here (default: same as '
                '--download, or "..")',
            default=None)

    parser.add_argument('--keep', '-k',
            help='Keep temporary built tree after a successful build',
            action='store_true', default=False)
    parser.add_argument('--download', '-d',
            help='Download build products to DOWNLOAD/ (existing files will '
                'be overwritten; gbp users might use "../build-area")',
            default=None)
    parser.add_argument('--install', '-i',
            help='Upgrade build products on remote host',
            action='store_true', default=False)
    parser.add_argument('--force-local-install',
            help='Force -i or -I to work even if building on localhost',
            action='store_true', default=False)
    parser.add_argument('--install-all', '-I',
            help='Install build products on remote host even if not already '
            'installed',
            action='store_true', default=False)

    parser.add_argument('-j', '--jobs', type=int,
            help='Run this many jobs in parallel', default=5)

    parser.add_argument('--merge', action='append', metavar='COMMIT',
            dest='merges', default=[],
            help='merge Git branches or tags, for example "--merge origin/wip"')
    parser.add_argument('--merge-strategy', metavar='STRATEGY',
            dest='merge_strategy', default='recursive',
            help='Use "git merge -s STRATEGY"')
    parser.add_argument('--merge-option', metavar='OPTION',
            dest='merge_options', action='append', default=[],
            help='Use "git merge -X OPTION [-X OPTION2...]"')
    parser.add_argument('--add-remote', action='append', metavar='NAME=URI',
            dest='remotes', default=[],
            help='add extra Git remotes for use with --merge, in the format '
            '"--add-remote upstream=https://git.example.com/hello"')

    parser.add_argument('builder', metavar='HOSTNAME',
            help='Remote host to use for the build (no default, use '
            '"localhost" for a local build)')

    args = parser.parse_args()

    if args.orig_dir is None:
        if args.download is None:
            args.orig_dir = '..'
        else:
            args.orig_dir = args.download

    if args.source_only:
        args.source = True

    SnapshotBuilder(args).main()

if __name__ == '__main__':
    main()
