added new command 'osc distributions'

[opensuse:osc.git] / osc / core.py

# Copyright (C) 2006 Novell Inc.  All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or version 3 (at your option).

__version__ = '0.128git'

# __store_version__ is to be incremented when the format of the working copy
# "store" changes in an incompatible way. Please add any needed migration
# functionality to check_store_version().
__store_version__ = '1.0'

import os
import os.path
import sys
import urllib2
from urllib import pathname2url, quote_plus, urlencode, unquote
from urlparse import urlsplit, urlunsplit
from cStringIO import StringIO
import shutil
import oscerr
import conf
import subprocess
import re
import socket
try:
    from xml.etree import cElementTree as ET
except ImportError:
    import cElementTree as ET



DISTURL_RE = re.compile(r"^(?P<bs>.*)://(?P<apiurl>.*?)/(?P<project>.*?)/(?P<repository>.*?)/(?P<revision>.*)-(?P<source>.*)$")
BUILDLOGURL_RE = re.compile(r"^(?P<apiurl>https?://.*?)/build/(?P<project>.*?)/(?P<repository>.*?)/(?P<arch>.*?)/(?P<package>.*?)/_log$")
BUFSIZE = 1024*1024
store = '.osc'

# NOTE: do not use this anymore, use conf.exclude_glob instead.
# but this needs to stay to avoid breakage of tools which use osc lib
exclude_stuff = [store, 'CVS', '*~', '#*#', '.*', '_linkerror']

new_project_templ = """\
<project name="%(name)s">

  <title></title> <!-- Short title of NewProject -->
  <description>
    <!-- This is for a longer description of the purpose of the project -->
  </description>

  <person role="maintainer" userid="%(user)s" />
  <person role="bugowner" userid="%(user)s" />
<!-- remove this block to publish your packages on the mirrors -->
  <publish>
    <disable />
  </publish>
  <build>
    <enable />
  </build>
  <debuginfo>
    <disable />
  </debuginfo>

<!-- remove this comment to enable one or more build targets

  <repository name="openSUSE_Factory">
    <path project="openSUSE:Factory" repository="standard" />
    <arch>x86_64</arch>
    <arch>i586</arch>
  </repository>
  <repository name="openSUSE_11.2">
    <path project="openSUSE:11.2" repository="standard"/>
    <arch>x86_64</arch>
    <arch>i586</arch>
  </repository>
  <repository name="openSUSE_11.1">
    <path project="openSUSE:11.1" repository="standard"/>
    <arch>x86_64</arch>
    <arch>i586</arch>
  </repository>
  <repository name="Fedora_12">
    <path project="Fedora:12" repository="standard" />
    <arch>x86_64</arch>
    <arch>i586</arch>
  </repository>
  <repository name="SLE_11">
    <path project="SUSE:SLE-11" repository="standard" />
    <arch>x86_64</arch>
    <arch>i586</arch>
  </repository>
-->

</project>
"""

new_package_templ = """\
<package name="%(name)s">

  <title></title> <!-- Title of package -->

  <description>
<!-- for long description -->
  </description>

  <person role="maintainer" userid="%(user)s"/>
  <person role="bugowner" userid="%(user)s"/>
<!--
  <url>PUT_UPSTREAM_URL_HERE</url>
-->

<!--
  use one of the examples below to disable building of this package
  on a certain architecture, in a certain repository,
  or a combination thereof:

  <disable arch="x86_64"/>
  <disable repository="SUSE_SLE-10"/>
  <disable repository="SUSE_SLE-10" arch="x86_64"/>

  Possible sections where you can use the tags above:
  <build>
  </build>
  <debuginfo>
  </debuginfo>
  <publish>
  </publish>
  <useforbuild>
  </useforbuild>

  Please have a look at:
  http://wiki.opensuse.org/Restricted_formats
  Packages containing formats listed there are NOT allowed to
  be packaged in the openSUSE Buildservice and will be deleted!

-->

</package>
"""

new_attribute_templ = """\
<attributes>
  <attribute namespace="" name="">
    <value><value>
  </attribute>
</attributes>
"""

new_user_template = """\
<person>
  <login>%(user)s</login>
  <email>PUT_EMAIL_ADDRESS_HERE</email>
  <realname>PUT_REAL_NAME_HERE</realname>
  <watchlist>
    <project name="home:%(user)s"/>
  </watchlist>
</person>
"""

info_templ = """\
Project name: %s
Package name: %s
Path: %s
API URL: %s
Source URL: %s
srcmd5: %s
Revision: %s
Link info: %s
"""

new_pattern_template = """\
<!-- See http://svn.opensuse.org/svn/zypp/trunk/libzypp/zypp/parser/yum/schema/patterns.rng -->

<pattern>
</pattern>
"""

buildstatus_symbols = {'succeeded':       '.',
                       'disabled':        ' ',
                       'expansion error': 'U',  # obsolete with OBS 2.0
                       'unresolvable':    'U',
                       'failed':          'F',
                       'broken':          'B',
                       'blocked':         'b',
                       'building':        '%',
                       'finished':        'f',
                       'scheduled':       's',
                       'excluded':        'x',
                       'dispatching':     'd',
}


# our own xml writer function to write xml nice, but with correct syntax
# This function is from http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/
from xml.dom import minidom
def fixed_writexml(self, writer, indent="", addindent="", newl=""):
    # indent = current indentation
    # addindent = indentation to add to higher levels
    # newl = newline string
    writer.write(indent+"<" + self.tagName)

    attrs = self._get_attributes()
    a_names = attrs.keys()
    a_names.sort()

    for a_name in a_names:
        writer.write(" %s=\"" % a_name)
        minidom._write_data(writer, attrs[a_name].value)
        writer.write("\"")
    if self.childNodes:
        if len(self.childNodes) == 1 \
          and self.childNodes[0].nodeType == minidom.Node.TEXT_NODE:
            writer.write(">")
            self.childNodes[0].writexml(writer, "", "", "")
            writer.write("</%s>%s" % (self.tagName, newl))
            return
        writer.write(">%s"%(newl))
        for node in self.childNodes:
            node.writexml(writer,indent+addindent,addindent,newl)
        writer.write("%s</%s>%s" % (indent,self.tagName,newl))
    else:
        writer.write("/>%s"%(newl))
# replace minidom's function with ours
minidom.Element.writexml = fixed_writexml


# os.path.samefile is available only under Unix
def os_path_samefile(path1, path2):
    try:
        return os.path.samefile(path1, path2)
    except:
        return os.path.realpath(path1) == os.path.realpath(path2)

class File:
    """represent a file, including its metadata"""
    def __init__(self, name, md5, size, mtime):
        self.name = name
        self.md5 = md5
        self.size = size
        self.mtime = mtime
    def __str__(self):
        return self.name


class Serviceinfo:
    """Source service content
    """
    def __init__(self):
        """creates an empty serviceinfo instance"""
        self.commands = None

    def read(self, serviceinfo_node):
        """read in the source services <services> element passed as
        elementtree node.
        """
        if serviceinfo_node == None:
            return
        self.commands = []
        services = serviceinfo_node.findall('service')

        for service in services:
            name = service.get('name')
            try:
                for param in service.findall('param'):
                    option = param.get('name', None)
                    value = param.text
                    name += " --" + option + " '" + value + "'"
                self.commands.append(name)
            except:
                msg = 'invalid service format:\n%s' % ET.tostring(serviceinfo_node)
                raise oscerr.APIError(msg)

    def addVerifyFile(self, serviceinfo_node, filename):
        import hashlib

        f = open(filename, 'r')
        digest = hashlib.sha256(f.read()).hexdigest()
        f.close()

        r = serviceinfo_node
        s = ET.Element( "service", name="verify_file" )
        ET.SubElement(s, "param", name="file").text = filename
        ET.SubElement(s, "param", name="verifier").text  = "sha256"
        ET.SubElement(s, "param", name="checksum").text = digest

        r.append( s )
        return r


    def addDownloadUrl(self, serviceinfo_node, url_string):
        from urlparse import urlparse
        url = urlparse( url_string )
        protocol = url.scheme
        host = url.netloc
        path = url.path

        r = serviceinfo_node
        s = ET.Element( "service", name="download_url" )
        ET.SubElement(s, "param", name="protocol").text = protocol
        ET.SubElement(s, "param", name="host").text     = host
        ET.SubElement(s, "param", name="path").text     = path

        r.append( s )
        return r


    def execute(self, dir):
        import tempfile

        for call in self.commands:
            temp_dir = tempfile.mkdtemp()
            name = call.split(None, 1)[0]
            if not os.path.exists("/usr/lib/obs/service/"+name):
                msg =  "ERROR: service is not installed!\n"
                msg += "Maybe try this: zypper in obs-service-" + name
                raise oscerr.APIError(msg)
            c = "/usr/lib/obs/service/" + call + " --outdir " + temp_dir
            if conf.config['verbose'] > 1:
                print "Run source service:", c
            ret = subprocess.call(c, shell=True)
            if ret != 0:
                print "ERROR: service call failed: " + c
                # FIXME: addDownloadUrlService calls si.execute after 
                #        updating _services.
                print "       (your _services file may be corrupt now)"

            for file in os.listdir(temp_dir):
                shutil.move( os.path.join(temp_dir, file), os.path.join(dir, "_service:"+name+":"+file) )
            os.rmdir(temp_dir)

class Linkinfo:
    """linkinfo metadata (which is part of the xml representing a directory
    """
    def __init__(self):
        """creates an empty linkinfo instance"""
        self.project = None
        self.package = None
        self.xsrcmd5 = None
        self.lsrcmd5 = None
        self.srcmd5 = None
        self.error = None
        self.rev = None
        self.baserev = None

    def read(self, linkinfo_node):
        """read in the linkinfo metadata from the <linkinfo> element passed as
        elementtree node.
        If the passed element is None, the method does nothing.
        """
        if linkinfo_node == None:
            return
        self.project = linkinfo_node.get('project')
        self.package = linkinfo_node.get('package')
        self.xsrcmd5 = linkinfo_node.get('xsrcmd5')
        self.lsrcmd5 = linkinfo_node.get('lsrcmd5')
        self.srcmd5  = linkinfo_node.get('srcmd5')
        self.error   = linkinfo_node.get('error')
        self.rev     = linkinfo_node.get('rev')
        self.baserev = linkinfo_node.get('baserev')

    def islink(self):
        """returns True if the linkinfo is not empty, otherwise False"""
        if self.xsrcmd5 or self.lsrcmd5:
            return True
        return False

    def isexpanded(self):
        """returns True if the package is an expanded link"""
        if self.lsrcmd5 and not self.xsrcmd5:
            return True
        return False

    def haserror(self):
        """returns True if the link is in error state (could not be applied)"""
        if self.error:
            return True
        return False

    def __str__(self):
        """return an informatory string representation"""
        if self.islink() and not self.isexpanded():
            return 'project %s, package %s, xsrcmd5 %s, rev %s' \
                    % (self.project, self.package, self.xsrcmd5, self.rev)
        elif self.islink() and self.isexpanded():
            if self.haserror():
                return 'broken link to project %s, package %s, srcmd5 %s, lsrcmd5 %s: %s' \
                        % (self.project, self.package, self.srcmd5, self.lsrcmd5, self.error)
            else:
                return 'expanded link to project %s, package %s, srcmd5 %s, lsrcmd5 %s' \
                        % (self.project, self.package, self.srcmd5, self.lsrcmd5)
        else:
            return 'None'


class Project:
    """represent a project directory, holding packages"""
    def __init__(self, dir, getPackageList=True, progress_obj=None):
        import fnmatch
        self.dir = dir
        self.absdir = os.path.abspath(dir)
        self.progress_obj = progress_obj

        self.name = store_read_project(self.dir)
        self.apiurl = store_read_apiurl(self.dir)

        if getPackageList:
            self.pacs_available = meta_get_packagelist(self.apiurl, self.name)
        else:
            self.pacs_available = []

        if conf.config['do_package_tracking']:
            self.pac_root = self.read_packages().getroot()
            self.pacs_have = [ pac.get('name') for pac in self.pac_root.findall('package') ]
            self.pacs_excluded = [ i for i in os.listdir(self.dir)
                                   for j in conf.config['exclude_glob']
                                   if fnmatch.fnmatch(i, j) ]
            self.pacs_unvers = [ i for i in os.listdir(self.dir) if i not in self.pacs_have and i not in self.pacs_excluded ]
            # store all broken packages (e.g. packages which where removed by a non-osc cmd)
            # in the self.pacs_broken list
            self.pacs_broken = []
            for p in self.pacs_have:
                if not os.path.isdir(os.path.join(self.absdir, p)):
                    # all states will be replaced with the '!'-state
                    # (except it is already marked as deleted ('D'-state))
                    self.pacs_broken.append(p)
        else:
            self.pacs_have = [ i for i in os.listdir(self.dir) if i in self.pacs_available ]

        self.pacs_missing = [ i for i in self.pacs_available if i not in self.pacs_have ]

    def checkout_missing_pacs(self, expand_link=False):
        for pac in self.pacs_missing:

            if conf.config['do_package_tracking'] and pac in self.pacs_unvers:
                # pac is not under version control but a local file/dir exists
                msg = 'can\'t add package \'%s\': Object already exists' % pac
                raise oscerr.PackageExists(self.name, pac, msg)
            else:
                print 'checking out new package %s' % pac
                checkout_package(self.apiurl, self.name, pac, \
                                 pathname=getTransActPath(os.path.join(self.dir, pac)), \
                                 prj_obj=self, prj_dir=self.dir, expand_link=expand_link, progress_obj=self.progress_obj)

    def set_state(self, pac, state):
        node = self.get_package_node(pac)
        if node == None:
            self.new_package_entry(pac, state)
        else:
            node.attrib['state'] = state

    def get_package_node(self, pac):
        for node in self.pac_root.findall('package'):
            if pac == node.get('name'):
                return node
        return None

    def del_package_node(self, pac):
        for node in self.pac_root.findall('package'):
            if pac == node.get('name'):
                self.pac_root.remove(node)

    def get_state(self, pac):
        node = self.get_package_node(pac)
        if node != None:
            return node.get('state')
        else:
            return None

    def new_package_entry(self, name, state):
        ET.SubElement(self.pac_root, 'package', name=name, state=state)

    def read_packages(self):
        packages_file = os.path.join(self.absdir, store, '_packages')
        if os.path.isfile(packages_file) and os.path.getsize(packages_file):
            return ET.parse(packages_file)
        else:
            # scan project for existing packages and migrate them
            cur_pacs = []
            for data in os.listdir(self.dir):
                pac_dir = os.path.join(self.absdir, data)
                # we cannot use self.pacs_available because we cannot guarantee that the package list
                # was fetched from the server
                if data in meta_get_packagelist(self.apiurl, self.name) and is_package_dir(pac_dir) \
                   and Package(pac_dir).name == data:
                    cur_pacs.append(ET.Element('package', name=data, state=' '))
            store_write_initial_packages(self.absdir, self.name, cur_pacs)
            return ET.parse(os.path.join(self.absdir, store, '_packages'))

    def write_packages(self):
        # TODO: should we only modify the existing file instead of overwriting?
        ET.ElementTree(self.pac_root).write(os.path.join(self.absdir, store, '_packages'))

    def addPackage(self, pac):
        import fnmatch
        for i in conf.config['exclude_glob']:
            if fnmatch.fnmatch(pac, i):
                msg = 'invalid package name: \'%s\' (see \'exclude_glob\' config option)' % pac
                raise oscerr.OscIOError(None, msg)
        state = self.get_state(pac)
        if state == None or state == 'D':
            self.new_package_entry(pac, 'A')
            self.write_packages()
            # sometimes the new pac doesn't exist in the list because
            # it would take too much time to update all data structs regularly
            if pac in self.pacs_unvers:
                self.pacs_unvers.remove(pac)
        else:
            raise oscerr.PackageExists(self.name, pac, 'package \'%s\' is already under version control' % pac)

    def delPackage(self, pac, force = False):
        state = self.get_state(pac.name)
        can_delete = True
        if state == ' ' or state == 'D':
            del_files = []
            for file in pac.filenamelist + pac.filenamelist_unvers:
                filestate = pac.status(file)
                if filestate == 'M' or filestate == 'C' or \
                   filestate == 'A' or filestate == '?':
                    can_delete = False
                else:
                    del_files.append(file)
            if can_delete or force:
                for file in del_files:
                    pac.delete_localfile(file)
                    if pac.status(file) != '?':
                        pac.delete_storefile(file)
                        # this is not really necessary
                        pac.put_on_deletelist(file)
                        print statfrmt('D', getTransActPath(os.path.join(pac.dir, file)))
                print statfrmt('D', getTransActPath(os.path.join(pac.dir, os.pardir, pac.name)))
                pac.write_deletelist()
                self.set_state(pac.name, 'D')
                self.write_packages()
            else:
                print 'package \'%s\' has local modifications (see osc st for details)' % pac.name
        elif state == 'A':
            if force:
                delete_dir(pac.absdir)
                self.del_package_node(pac.name)
                self.write_packages()
                print statfrmt('D', pac.name)
            else:
                print 'package \'%s\' has local modifications (see osc st for details)' % pac.name
        elif state == None:
            print 'package is not under version control'
        else:
            print 'unsupported state'

    def update(self, pacs = (), expand_link=False, unexpand_link=False, service_files=False):
        if len(pacs):
            for pac in pacs:
                Package(os.path.join(self.dir, pac, progress_obj=self.progress_obj)).update()
        else:
            # we need to make sure that the _packages file will be written (even if an exception
            # occurs)
            try:
                # update complete project
                # packages which no longer exists upstream
                upstream_del = [ pac for pac in self.pacs_have if not pac in self.pacs_available and self.get_state(pac) != 'A']

                for pac in upstream_del:
                    p = Package(os.path.join(self.dir, pac))
                    self.delPackage(p, force = True)
                    delete_storedir(p.storedir)
                    try:
                        os.rmdir(pac)
                    except:
                        pass
                    self.pac_root.remove(self.get_package_node(p.name))
                    self.pacs_have.remove(pac)

                for pac in self.pacs_have:
                    state = self.get_state(pac)
                    if pac in self.pacs_broken:
                        if self.get_state(pac) != 'A':
                            checkout_package(self.apiurl, self.name, pac,
                                             pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, \
                                             prj_dir=self.dir, expand_link=not unexpand_link, progress_obj=self.progress_obj)
                    elif state == ' ':
                        # do a simple update
                        p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj)
                        rev = None
                        if expand_link and p.islink() and not p.isexpanded():
                            if p.haslinkerror():
                                try:
                                    rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev)
                                except:
                                    rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev, linkrev="base")
                                    p.mark_frozen()
                            else:
                                rev = p.linkinfo.xsrcmd5
                            print 'Expanding to rev', rev
                        elif unexpand_link and p.islink() and p.isexpanded():
                            rev = p.linkinfo.lsrcmd5
                            print 'Unexpanding to rev', rev
                        elif p.islink() and p.isexpanded():
                            rev = p.latest_rev()
                        print 'Updating %s' % p.name
                        p.update(rev, service_files)
                        if unexpand_link:
                            p.unmark_frozen()
                    elif state == 'D':
                        # TODO: Package::update has to fixed to behave like svn does
                        if pac in self.pacs_broken:
                            checkout_package(self.apiurl, self.name, pac,
                                             pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, \
                                             prj_dir=self.dir, expand_link=expand_link, progress_obj=self.progress_obj)
                        else:
                            Package(os.path.join(self.dir, pac, progress_obj=self.progress_obj)).update()
                    elif state == 'A' and pac in self.pacs_available:
                        # file/dir called pac already exists and is under version control
                        msg = 'can\'t add package \'%s\': Object already exists' % pac
                        raise oscerr.PackageExists(self.name, pac, msg)
                    elif state == 'A':
                        # do nothing
                        pass
                    else:
                        print 'unexpected state.. package \'%s\'' % pac

                self.checkout_missing_pacs(expand_link=not unexpand_link)
            finally:
                self.write_packages()

    def commit(self, pacs = (), msg = '', files = {}, validators = None, verbose_validation = None):
        if len(pacs):
            try:
                for pac in pacs:
                    todo = []
                    if files.has_key(pac):
                        todo = files[pac]
                    state = self.get_state(pac)
                    if state == 'A':
                        self.commitNewPackage(pac, msg, todo, validators=validators, verbose_validation=verbose_validation)
                    elif state == 'D':
                        self.commitDelPackage(pac)
                    elif state == ' ':
                        # display the correct dir when sending the changes
                        if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()):
                            p = Package('.')
                        else:
                            p = Package(os.path.join(self.dir, pac))
                        p.todo = todo
                        p.commit(msg, validators=validators, verbose_validation=verbose_validation)
                    elif pac in self.pacs_unvers and not is_package_dir(os.path.join(self.dir, pac)):
                        print 'osc: \'%s\' is not under version control' % pac
                    elif pac in self.pacs_broken:
                        print 'osc: \'%s\' package not found' % pac
                    elif state == None:
                        self.commitExtPackage(pac, msg, todo)
            finally:
                self.write_packages()
        else:
            # if we have packages marked as '!' we cannot commit
            for pac in self.pacs_broken:
                if self.get_state(pac) != 'D':
                    msg = 'commit failed: package \'%s\' is missing' % pac
                    raise oscerr.PackageMissing(self.name, pac, msg)
            try:
                for pac in self.pacs_have:
                    state = self.get_state(pac)
                    if state == ' ':
                        # do a simple commit
                        Package(os.path.join(self.dir, pac)).commit(msg, validators=validators, verbose_validation=verbose_validation)
                    elif state == 'D':
                        self.commitDelPackage(pac)
                    elif state == 'A':
                        self.commitNewPackage(pac, msg, validators=validators, verbose_validation=verbose_validation)
            finally:
                self.write_packages()

    def commitNewPackage(self, pac, msg = '', files = [], validators = None, verbose_validation = None):
        """creates and commits a new package if it does not exist on the server"""
        if pac in self.pacs_available:
            print 'package \'%s\' already exists' % pac
        else:
            user = conf.get_apiurl_usr(self.apiurl)
            edit_meta(metatype='pkg',
                      path_args=(quote_plus(self.name), quote_plus(pac)),
                      template_args=({
                              'name': pac,
                              'user': user}),
                      apiurl=self.apiurl)
            # display the correct dir when sending the changes
            olddir = os.getcwd()
            if os_path_samefile(os.path.join(self.dir, pac), os.curdir):
                os.chdir(os.pardir)
                p = Package(pac)
            else:
                p = Package(os.path.join(self.dir, pac))
            p.todo = files
            print statfrmt('Sending', os.path.normpath(p.dir))
            p.commit(msg=msg, validators=validators, verbose_validation=verbose_validation)
            self.set_state(pac, ' ')
            os.chdir(olddir)

    def commitDelPackage(self, pac):
        """deletes a package on the server and in the working copy"""
        try:
            # display the correct dir when sending the changes
            if os_path_samefile(os.path.join(self.dir, pac), os.curdir):
                pac_dir = pac
            else:
                pac_dir = os.path.join(self.dir, pac)
            p = Package(os.path.join(self.dir, pac))
            #print statfrmt('Deleting', os.path.normpath(os.path.join(p.dir, os.pardir, pac)))
            delete_storedir(p.storedir)
            try:
                os.rmdir(p.dir)
            except:
                pass
        except OSError:
            pac_dir = os.path.join(self.dir, pac)
        #print statfrmt('Deleting', getTransActPath(os.path.join(self.dir, pac)))
        print statfrmt('Deleting', getTransActPath(pac_dir))
        delete_package(self.apiurl, self.name, pac)
        self.del_package_node(pac)

    def commitExtPackage(self, pac, msg, files = []):
        """commits a package from an external project"""
        if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()):
            pac_path = '.'
        else:
            pac_path = os.path.join(self.dir, pac)

        project = store_read_project(pac_path)
        package = store_read_package(pac_path)
        apiurl = store_read_apiurl(pac_path)
        if meta_exists(metatype='pkg',
                       path_args=(quote_plus(project), quote_plus(package)),
                       template_args=None,
                       create_new=False, apiurl=apiurl):
            p = Package(pac_path)
            p.todo = files
            p.commit(msg)
        else:
            user = conf.get_apiurl_usr(self.apiurl)
            edit_meta(metatype='pkg',
                      path_args=(quote_plus(project), quote_plus(package)),
                      template_args=({
                              'name': pac,
                              'user': user}),
                              apiurl=apiurl)
            p = Package(pac_path)
            p.todo = files
            p.commit(msg)

    def __str__(self):
        r = []
        r.append('*****************************************************')
        r.append('Project %s (dir=%s, absdir=%s)' % (self.name, self.dir, self.absdir))
        r.append('have pacs:\n%s' % ', '.join(self.pacs_have))
        r.append('missing pacs:\n%s' % ', '.join(self.pacs_missing))
        r.append('*****************************************************')
        return '\n'.join(r)



class Package:
    """represent a package (its directory) and read/keep/write its metadata"""
    def __init__(self, workingdir, progress_obj=None, limit_size=None):
        self.dir = workingdir
        self.absdir = os.path.abspath(self.dir)
        self.storedir = os.path.join(self.absdir, store)
        self.progress_obj = progress_obj
        self.limit_size = limit_size
        if limit_size and limit_size == 0:
           self.limit_size = None

        check_store_version(self.dir)

        self.prjname = store_read_project(self.dir)
        self.name = store_read_package(self.dir)
        self.apiurl = store_read_apiurl(self.dir)

        self.update_datastructs()

        self.todo = []
        self.todo_send = []
        self.todo_delete = []

    def info(self):
        source_url = makeurl(self.apiurl, ['source', self.prjname, self.name])
        r = info_templ % (self.prjname, self.name, self.absdir, self.apiurl, source_url, self.srcmd5, self.rev, self.linkinfo)
        return r

    def addfile(self, n):
        st = os.stat(os.path.join(self.dir, n))
        shutil.copyfile(os.path.join(self.dir, n), os.path.join(self.storedir, n))

    def delete_file(self, n, force=False):
        """deletes a file if possible and marks the file as deleted"""
        state = '?'
        try:
            state = self.status(n)
        except IOError, ioe:
            if not force:
                raise ioe
        if state in ['?', 'A', 'M'] and not force:
            return (False, state)
        self.delete_localfile(n)
        if state != 'A':
            self.put_on_deletelist(n)
            self.write_deletelist()
        else:
            self.delete_storefile(n)
        return (True, state)

    def delete_storefile(self, n):
        try: os.unlink(os.path.join(self.storedir, n))
        except: pass

    def delete_localfile(self, n):
        try: os.unlink(os.path.join(self.dir, n))
        except: pass

    def put_on_deletelist(self, n):
        if n not in self.to_be_deleted:
            self.to_be_deleted.append(n)

    def put_on_conflictlist(self, n):
        if n not in self.in_conflict:
            self.in_conflict.append(n)

    def clear_from_conflictlist(self, n):
        """delete an entry from the file, and remove the file if it would be empty"""
        if n in self.in_conflict:

            filename = os.path.join(self.dir, n)
            storefilename = os.path.join(self.storedir, n)
            myfilename = os.path.join(self.dir, n + '.mine')
            if self.islinkrepair() or self.ispulled():
                upfilename = os.path.join(self.dir, n + '.new')
            else:
                upfilename = os.path.join(self.dir, n + '.r' + self.rev)

            try:
                os.unlink(myfilename)
                # the working copy may be updated, so the .r* ending may be obsolete...
                # then we don't care
                os.unlink(upfilename)
                if self.islinkrepair() or self.ispulled():
                    os.unlink(os.path.join(self.dir, n + '.old'))
            except:
                pass

            self.in_conflict.remove(n)

            self.write_conflictlist()

    # XXX: this isn't used at all
    def write_meta_mode(self):
        # XXX: the "elif" is somehow a contradiction (with current and the old implementation
        #      it's not possible to "leave" the metamode again) (except if you modify pac.meta
        #      which is really ugly:) )
        if self.meta:
            store_write_string(self.absdir, '_meta_mode', '')
        elif self.ismetamode():
            os.unlink(os.path.join(self.storedir, '_meta_mode'))

    def write_sizelimit(self):
        if self.size_limit and self.size_limit <= 0:
            try:
                os.unlink(os.path.join(self.storedir, '_size_limit'))
            except:
                pass
        else:
            fname = os.path.join(self.storedir, '_size_limit')
            f = open(fname, 'w')
            f.write(str(self.size_limit))
            f.close()

    def write_deletelist(self):
        if len(self.to_be_deleted) == 0:
            try:
                os.unlink(os.path.join(self.storedir, '_to_be_deleted'))
            except:
                pass
        else:
            fname = os.path.join(self.storedir, '_to_be_deleted')
            f = open(fname, 'w')
            f.write('\n'.join(self.to_be_deleted))
            f.write('\n')
            f.close()

    def delete_source_file(self, n):
        """delete local a source file"""
        self.delete_localfile(n)
        self.delete_storefile(n)

    def delete_remote_source_file(self, n):
        """delete a remote source file (e.g. from the server)"""
        query = 'rev=upload'
        u = makeurl(self.apiurl, ['source', self.prjname, self.name, pathname2url(n)], query=query)
        http_DELETE(u)

    def put_source_file(self, n):

        # escaping '+' in the URL path (note: not in the URL query string) is
        # only a workaround for ruby on rails, which swallows it otherwise
        query = 'rev=upload'
        u = makeurl(self.apiurl, ['source', self.prjname, self.name, pathname2url(n)], query=query)
        http_PUT(u, file = os.path.join(self.dir, n))

        shutil.copyfile(os.path.join(self.dir, n), os.path.join(self.storedir, n))

    def commit(self, msg='', validators=None, verbose_validation=None):
        # commit only if the upstream revision is the same as the working copy's
        upstream_rev = self.latest_rev()
        if self.rev != upstream_rev:
            raise oscerr.WorkingCopyOutdated((self.absdir, self.rev, upstream_rev))

        if not self.todo:
            self.todo = self.filenamelist_unvers + self.filenamelist

        pathn = getTransActPath(self.dir)

        if validators and not self.name.startswith('_'):
            import subprocess
            import stat
            for validator in sorted(os.listdir(validators)):
                if validator.startswith('.'):
                   continue
                fn = os.path.join(validators, validator)
                mode = os.stat(fn).st_mode
                if stat.S_ISREG(mode):
                   if verbose_validation:
                       print "osc runs source service:", fn
                       p = subprocess.Popen([fn, "--verbose"], close_fds=True)
                   else:
                       p = subprocess.Popen([fn], close_fds=True)
                   if p.wait() != 0:
                       raise oscerr.RuntimeError(p.stdout, validator )

        have_conflicts = False
        for filename in self.todo:
            if not filename.startswith('_service:') and not filename.startswith('_service_'):
                st = self.status(filename)
                if st == 'S':
                    self.todo.remove(filename)
                elif st == 'A' or st == 'M':
                    self.todo_send.append(filename)
                    print statfrmt('Sending', os.path.join(pathn, filename))
                elif st == 'D':
                    self.todo_delete.append(filename)
                    print statfrmt('Deleting', os.path.join(pathn, filename))
                elif st == 'C':
                    have_conflicts = True

        if have_conflicts:
            print 'Please resolve all conflicts before committing using "osc resolved FILE"!'
            return 1

        if not self.todo_send and not self.todo_delete and not self.rev == "upload" and not self.islinkrepair() and not self.ispulled():
            print 'nothing to do for package %s' % self.name
            return 1

        if self.islink() and self.isexpanded():
            # resolve the link into the upload revision
            # XXX: do this always?
            query = { 'cmd': 'copy', 'rev': 'upload', 'orev': self.rev }
            u = makeurl(self.apiurl, ['source', self.prjname, self.name], query=query)
            f = http_POST(u)

        print 'Transmitting file data ',
        try:
            for filename in self.todo_delete:
                # do not touch local files on commit --
                # delete remotely instead
                self.delete_remote_source_file(filename)
                self.to_be_deleted.remove(filename)
            for filename in self.todo_send:
                sys.stdout.write('.')
                sys.stdout.flush()
                self.put_source_file(filename)

            # all source files are committed - now comes the log
            query = { 'cmd'    : 'commit',
                      'rev'    : 'upload',
                      'user'   : conf.get_apiurl_usr(self.apiurl),
                      'comment': msg }
            if self.islink() and self.isexpanded():
                query['keeplink'] = '1'
                if conf.config['linkcontrol'] or self.isfrozen():
                    query['linkrev'] = self.linkinfo.srcmd5
                if self.ispulled():
                    query['repairlink'] = '1'
                    query['linkrev'] = self.get_pulled_srcmd5()
            if self.islinkrepair():
                query['repairlink'] = '1'
            u = makeurl(self.apiurl, ['source', self.prjname, self.name], query=query)
            f = http_POST(u)
        except Exception, e:
            # delete upload revision
            try:
                query = { 'cmd': 'deleteuploadrev' }
                u = makeurl(self.apiurl, ['source', self.prjname, self.name], query=query)
                f = http_POST(u)
            except:
                pass
            raise e

        root = ET.parse(f).getroot()
        self.rev = int(root.get('rev'))
        print
        print 'Committed revision %s.' % self.rev

        if self.ispulled():
            os.unlink(os.path.join(self.storedir, '_pulled'))
        if self.islinkrepair():
            os.unlink(os.path.join(self.storedir, '_linkrepair'))
            self.linkrepair = False
            # XXX: mark package as invalid?
            print 'The source link has been repaired. This directory can now be removed.'
        if self.islink() and self.isexpanded():
            self.update_local_filesmeta(revision=self.latest_rev())
        else:
            self.update_local_filesmeta()
        self.write_deletelist()
        self.update_datastructs()

        if self.filenamelist.count('_service'):
            print 'The package contains a source service.'
            for filename in self.todo:
                if filename.startswith('_service:') and os.path.exists(filename):
                    os.unlink(filename) # remove local files
        print_request_list(self.apiurl, self.prjname, self.name)

    def write_conflictlist(self):
        if len(self.in_conflict) == 0:
            try:
                os.unlink(os.path.join(self.storedir, '_in_conflict'))
            except:
                pass
        else:
            fname = os.path.join(self.storedir, '_in_conflict')
            f = open(fname, 'w')
            f.write('\n'.join(self.in_conflict))
            f.write('\n')
            f.close()

    def updatefile(self, n, revision):
        filename = os.path.join(self.dir, n)
        storefilename = os.path.join(self.storedir, n)
        mtime = self.findfilebyname(n).mtime

        get_source_file(self.apiurl, self.prjname, self.name, n, targetfilename=filename,
                revision=revision, progress_obj=self.progress_obj, mtime=mtime, meta=self.meta)

        shutil.copyfile(filename, storefilename)

    def mergefile(self, n):
        filename = os.path.join(self.dir, n)
        storefilename = os.path.join(self.storedir, n)
        myfilename = os.path.join(self.dir, n + '.mine')
        upfilename = os.path.join(self.dir, n + '.r' + self.rev)
        os.rename(filename, myfilename)

        mtime = self.findfilebyname(n).mtime
        get_source_file(self.apiurl, self.prjname, self.name, n,
                        revision=self.rev, targetfilename=upfilename,
                        progress_obj=self.progress_obj, mtime=mtime, meta=self.meta)

        if binary_file(myfilename) or binary_file(upfilename):
            # don't try merging
            shutil.copyfile(upfilename, filename)
            shutil.copyfile(upfilename, storefilename)
            self.in_conflict.append(n)
            self.write_conflictlist()
            return 'C'
        else:
            # try merging
            # diff3 OPTIONS... MINE OLDER YOURS
            merge_cmd = 'diff3 -m -E %s %s %s > %s' % (myfilename, storefilename, upfilename, filename)
            # we would rather use the subprocess module, but it is not availablebefore 2.4
            ret = subprocess.call(merge_cmd, shell=True)

            #   "An exit status of 0 means `diff3' was successful, 1 means some
            #   conflicts were found, and 2 means trouble."
            if ret == 0:
                # merge was successful... clean up
                shutil.copyfile(upfilename, storefilename)
                os.unlink(upfilename)
                os.unlink(myfilename)
                return 'G'
            elif ret == 1:
                # unsuccessful merge
                shutil.copyfile(upfilename, storefilename)
                self.in_conflict.append(n)
                self.write_conflictlist()
                return 'C'
            else:
                print >>sys.stderr, '\ndiff3 got in trouble... exit code:', ret
                print >>sys.stderr, 'the command line was:'
                print >>sys.stderr, merge_cmd
                sys.exit(1)



    def update_local_filesmeta(self, revision=None):
        """
        Update the local _files file in the store.
        It is replaced with the version pulled from upstream.
        """
        meta = ''.join(show_files_meta(self.apiurl, self.prjname, self.name, revision=revision, limit_size=self.limit_size, meta=self.meta))
        store_write_string(self.absdir, '_files', meta)

    def update_datastructs(self):
        """
        Update the internal data structures if the local _files
        file has changed (e.g. update_local_filesmeta() has been
        called).
        """
        import fnmatch
        files_tree = read_filemeta(self.dir)
        files_tree_root = files_tree.getroot()

        self.rev = files_tree_root.get('rev')
        self.srcmd5 = files_tree_root.get('srcmd5')

        self.linkinfo = Linkinfo()
        self.linkinfo.read(files_tree_root.find('linkinfo'))

        self.filenamelist = []
        self.filelist = []
        self.skipped = []
        for node in files_tree_root.findall('entry'):
            try:
                f = File(node.get('name'),
                         node.get('md5'),
                         int(node.get('size')),
                         int(node.get('mtime')))
                if node.get('skipped'):
                    self.skipped.append(f.name)
            except:
                # okay, a very old version of _files, which didn't contain any metadata yet...
                f = File(node.get('name'), '', 0, 0)
            self.filelist.append(f)
            self.filenamelist.append(f.name)

        self.to_be_deleted = read_tobedeleted(self.dir)
        self.in_conflict = read_inconflict(self.dir)
        self.linkrepair = os.path.isfile(os.path.join(self.storedir, '_linkrepair'))
        self.size_limit = read_sizelimit(self.dir)
        self.meta = self.ismetamode()

        # gather unversioned files, but ignore some stuff
        self.excluded = [ i for i in os.listdir(self.dir)
                          for j in conf.config['exclude_glob']
                          if fnmatch.fnmatch(i, j) ]
        self.filenamelist_unvers = [ i for i in os.listdir(self.dir)
                                     if i not in self.excluded
                                     if i not in self.filenamelist ]

    def islink(self):
        """tells us if the package is a link (has 'linkinfo').
        A package with linkinfo is a package which links to another package.
        Returns True if the package is a link, otherwise False."""
        return self.linkinfo.islink()

    def isexpanded(self):
        """tells us if the package is a link which is expanded.
        Returns True if the package is expanded, otherwise False."""
        return self.linkinfo.isexpanded()

    def islinkrepair(self):
        """tells us if we are repairing a broken source link."""
        return self.linkrepair

    def ispulled(self):
        """tells us if we have pulled a link."""
        return os.path.isfile(os.path.join(self.storedir, '_pulled'))

    def isfrozen(self):
        """tells us if the link is frozen."""
        return os.path.isfile(os.path.join(self.storedir, '_frozenlink'))

    def ismetamode(self):
        """tells us if the package is in meta mode"""
        return os.path.isfile(os.path.join(self.storedir, '_meta_mode'))

    def get_pulled_srcmd5(self):
        pulledrev = None
        for line in open(os.path.join(self.storedir, '_pulled'), 'r'):
            pulledrev = line.strip()
        return pulledrev

    def haslinkerror(self):
        """
        Returns True if the link is broken otherwise False.
        If the package is not a link it returns False.
        """
        return self.linkinfo.haserror()

    def linkerror(self):
        """
        Returns an error message if the link is broken otherwise None.
        If the package is not a link it returns None.
        """
        return self.linkinfo.error

    def update_local_pacmeta(self):
        """
        Update the local _meta file in the store.
        It is replaced with the version pulled from upstream.
        """
        meta = ''.join(show_package_meta(self.apiurl, self.prjname, self.name))
        store_write_string(self.absdir, '_meta', meta)

    def findfilebyname(self, n):
        for i in self.filelist:
            if i.name == n:
                return i

    def status(self, n):
        """
        status can be:

         file  storefile  file present  STATUS
        exists  exists      in _files

          x       x            -        'A'
          x       x            x        ' ' if digest differs: 'M'
                                            and if in conflicts file: 'C'
          x       -            -        '?'
          x       -            x        'D' and listed in _to_be_deleted
          -       x            x        '!'
          -       x            -        'D' (when file in working copy is already deleted)
          -       -            x        'F' (new in repo, but not yet in working copy)
          -       -            -        NOT DEFINED

        """

        known_by_meta = False
        exists = False
        exists_in_store = False
        if n in self.filenamelist:
            known_by_meta = True
        if os.path.exists(os.path.join(self.absdir, n)):
            exists = True
        if os.path.exists(os.path.join(self.storedir, n)):
            exists_in_store = True


        if n in self.skipped:
            state = 'S'
        elif exists and not exists_in_store and known_by_meta:
            state = 'D'
        elif n in self.to_be_deleted:
            state = 'D'
        elif n in self.in_conflict:
            state = 'C'
        elif exists and exists_in_store and known_by_meta:
            #print self.findfilebyname(n)
            if dgst(os.path.join(self.absdir, n)) != self.findfilebyname(n).md5:
                state = 'M'
            else:
                state = ' '
        elif exists and not exists_in_store and not known_by_meta:
            state = '?'
        elif exists and exists_in_store and not known_by_meta:
            state = 'A'
        elif not exists and exists_in_store and known_by_meta:
            state = '!'
        elif not exists and not exists_in_store and known_by_meta:
            state = 'F'
        elif not exists and exists_in_store and not known_by_meta:
            state = 'D'
        elif not exists and not exists_in_store and not known_by_meta:
            # this case shouldn't happen (except there was a typo in the filename etc.)
            raise IOError('osc: \'%s\' is not under version control' % n)

        return state

    def comparePac(self, cmp_pac):
        """
        This method compares the local filelist with
        the filelist of the passed package to see which files
        were added, removed and changed.
        """

        changed_files = []
        added_files = []
        removed_files = []

        for file in self.filenamelist+self.filenamelist_unvers:
            state = self.status(file)
            if file in self.skipped:
                continue
            if state == 'A' and (not file in cmp_pac.filenamelist):
                added_files.append(file)
            elif file in cmp_pac.filenamelist and state == 'D':
                removed_files.append(file)
            elif state == ' ' and not file in cmp_pac.filenamelist:
                added_files.append(file)
            elif file in cmp_pac.filenamelist and state != 'A' and state != '?':
                if dgst(os.path.join(self.absdir, file)) != cmp_pac.findfilebyname(file).md5:
                    changed_files.append(file)
        for file in cmp_pac.filenamelist:
            if not file in self.filenamelist:
                removed_files.append(file)
        removed_files = set(removed_files)

        return changed_files, added_files, removed_files

    def merge(self, otherpac):
        self.todo += otherpac.todo

    def __str__(self):
        r = """
name: %s
prjname: %s
workingdir: %s
localfilelist: %s
linkinfo: %s
rev: %s
'todo' files: %s
""" % (self.name,
        self.prjname,
        self.dir,
        '\n               '.join(self.filenamelist),
        self.linkinfo,
        self.rev,
        self.todo)

        return r


    def read_meta_from_spec(self, spec = None):
        import glob
        if spec:
            specfile = spec
        else:
            # scan for spec files
            speclist = glob.glob(os.path.join(self.dir, '*.spec'))
            if len(speclist) == 1:
                specfile = speclist[0]
            elif len(speclist) > 1:
                print 'the following specfiles were found:'
                for file in speclist:
                    print file
                print 'please specify one with --specfile'
                sys.exit(1)
            else:
                print 'no specfile was found - please specify one ' \
                      'with --specfile'
                sys.exit(1)

        data = read_meta_from_spec(specfile, 'Summary', 'Url', '%description')
        self.summary = data.get('Summary', '')
        self.url = data.get('Url', '')
        self.descr = data.get('%description', '')


    def update_package_meta(self, force=False):
        """
        for the updatepacmetafromspec subcommand
            argument force supress the confirm question
        """

        m = ''.join(show_package_meta(self.apiurl, self.prjname, self.name))

        root = ET.fromstring(m)
        root.find('title').text = self.summary
        root.find('description').text = ''.join(self.descr)
        url = root.find('url')
        if url == None:
            url = ET.SubElement(root, 'url')
        url.text = self.url

        u = makeurl(self.apiurl, ['source', self.prjname, self.name, '_meta'])
        mf = metafile(u, ET.tostring(root))

        if not force:
            print '*' * 36, 'old', '*' * 36
            print m
            print '*' * 36, 'new', '*' * 36
            print ET.tostring(root)
            print '*' * 72
            repl = raw_input('Write? (y/N/e) ')
        else:
            repl = 'y'

        if repl == 'y':
            mf.sync()
        elif repl == 'e':
            mf.edit()

        mf.discard()

    def mark_frozen(self):
        store_write_string(self.absdir, '_frozenlink', '')
        print
        print "The link in this package is currently broken. Checking"
        print "out the last working version instead; please use 'osc pull'"
        print "to repair the link."
        print

    def unmark_frozen(self):
        if os.path.exists(os.path.join(self.storedir, '_frozenlink')):
            os.unlink(os.path.join(self.storedir, '_frozenlink'))

    def latest_rev(self):
        if self.islinkrepair():
            upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrepair=1, meta=self.meta)
        elif self.islink() and self.isexpanded():
            if self.isfrozen() or self.ispulled():
                upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta)
            else:
                try:
                    upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, meta=self.meta)
                except:
                    try:
                        upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta)
                    except:
                        upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev="base", meta=self.meta)
                    self.mark_frozen()
        else:
            upstream_rev = show_upstream_rev(self.apiurl, self.prjname, self.name, meta=self.meta)
        return upstream_rev

    def update(self, rev = None, service_files = False, limit_size = None):
        # save filelist and (modified) status before replacing the meta file
        saved_filenames = self.filenamelist
        saved_modifiedfiles = [ f for f in self.filenamelist if self.status(f) == 'M' ]

        oldp = self
        if limit_size:
            self.limit_size = limit_size
        else:
            self.limit_size = read_sizelimit(self.dir)
        self.update_local_filesmeta(rev)
        self = Package(self.dir, progress_obj=self.progress_obj)

        # which files do no longer exist upstream?
        disappeared = [ f for f in saved_filenames if f not in self.filenamelist ]

        pathn = getTransActPath(self.dir)

        for filename in saved_filenames:
            if filename in self.skipped:
                continue
            if not filename.startswith('_service:') and filename in disappeared:
                print statfrmt('D', os.path.join(pathn, filename))
                # keep file if it has local modifications
                if oldp.status(filename) == ' ':
                    self.delete_localfile(filename)
                self.delete_storefile(filename)

        for filename in self.filenamelist:
            if filename in self.skipped:
                continue

            state = self.status(filename)
            if not service_files and filename.startswith('_service:'):
                pass
            elif state == 'M' and self.findfilebyname(filename).md5 == oldp.findfilebyname(filename).md5:
                # no merge necessary... local file is changed, but upstream isn't
                pass
            elif state == 'M' and filename in saved_modifiedfiles:
                status_after_merge = self.mergefile(filename)
                print statfrmt(status_after_merge, os.path.join(pathn, filename))
            elif state == 'M':
                self.updatefile(filename, rev)
                print statfrmt('U', os.path.join(pathn, filename))
            elif state == '!':
                self.updatefile(filename, rev)
                print 'Restored \'%s\'' % os.path.join(pathn, filename)
            elif state == 'F':
                self.updatefile(filename, rev)
                print statfrmt('A', os.path.join(pathn, filename))
            elif state == 'D' and self.findfilebyname(filename).md5 != oldp.findfilebyname(filename).md5:
                self.updatefile(filename, rev)
                self.delete_storefile(filename)
                print statfrmt('U', os.path.join(pathn, filename))
            elif state == ' ':
                pass

        self.update_local_pacmeta()

        #print ljust(p.name, 45), 'At revision %s.' % p.rev
        print 'At revision %s.' % self.rev

    def run_source_services(self):
        if self.filenamelist.count('_service'):
            service = ET.parse(os.path.join(self.absdir, '_service')).getroot()
            si = Serviceinfo()
            si.read(service)
            si.execute(self.absdir)

    def prepare_filelist(self):
        """Prepare a list of files, which will be processed by process_filelist
        method. This allows easy modifications of a file list in commit
        phase.
        """
        if not self.todo:
            self.todo = self.filenamelist + self.filenamelist_unvers
        self.todo.sort()

        ret = ""
        for f in [f for f in self.todo if not os.path.isdir(f)]:
            action = 'leave'
            status = self.status(f)
            if status == 'S':
                continue
            if status == '!':
                action = 'remove'
            ret += "%s %s %s\n" % (action, status, f)

        ret += """
# Edit a filelist for package \'%s\'
# Commands:
# l, leave = leave a file as is
# r, remove = remove a file
# a, add   = add a file
#
# If you remove file from a list, it will be unchanged
# If you remove all, commit will be aborted""" % self.name

        return ret

    def edit_filelist(self):
        """Opens a package list in editor for editing. This allows easy
        modifications of it just by simple text editing
        """

        import tempfile
        (fd, filename) = tempfile.mkstemp(prefix = 'osc-filelist', suffix = '.txt')
        f = os.fdopen(fd, 'w')
        f.write(self.prepare_filelist())
        f.close()
        mtime_orig = os.stat(filename).st_mtime

        while 1:
            run_editor(filename)
            mtime = os.stat(filename).st_mtime
            if mtime_orig < mtime:
                filelist = open(filename).readlines()
                os.unlink(filename)
                break
            else:
                raise oscerr.UserAbort()

        return self.process_filelist(filelist)

    def process_filelist(self, filelist):
        """Process a filelist - it add/remove or leave files. This depends on
        user input. If no file is processed, it raises an ValueError
        """

        loop = False
        for line in [l.strip() for l in filelist if (l[0] != "#" or l.strip() != '')]:

            foo = line.split(' ')
            if len(foo) == 4:
                action, state, name = (foo[0], ' ', foo[3])
            elif len(foo) == 3:
                action, state, name = (foo[0], foo[1], foo[2])
            else:
                break
            action = action.lower()
            loop = True

            if action in ('r', 'remove'):
                if self.status(name) == '?':
                    os.unlink(name)
                    if name in self.todo:
                        self.todo.remove(name)
                else:
                    self.delete_file(name, True)
            elif action in ('a', 'add'):
                if self.status(name) != '?':
                    print "Cannot add file %s with state %s, skipped" % (name, self.status(name))
                else:
                    self.addfile(name)
            elif action in ('l', 'leave'):
                pass
            else:
                raise ValueError("Unknow action `%s'" % action)

        if not loop:
            raise ValueError("Empty filelist")

class ReviewState:
    """for objects to represent the review state in a request"""
    def __init__(self, state=None, by_user=None, by_group=None, who=None, when=None, comment=None):
        self.state = state
        self.by_user  = by_user
        self.by_group = by_group
        self.who  = who
        self.when = when
        self.comment = comment

class RequestState:
    """for objects to represent the "state" of a request"""
    def __init__(self, name=None, who=None, when=None, comment=None):
        self.name = name
        self.who  = who
        self.when = when
        self.comment = comment

class Action:
    """represents an action"""
    def __init__(self, type, src_project, src_package, src_rev, dst_project, dst_package, src_update):
        self.type = type
        self.src_project = src_project
        self.src_package = src_package
        self.src_rev = src_rev
        self.dst_project = dst_project
        self.dst_package = dst_package
        self.src_update = src_update

class Request:
    """represent a request and holds its metadata
       it has methods to read in metadata from xml,
       different views, ..."""
    def __init__(self):
        self.reqid       = None
        self.state       = RequestState()
        self.who         = None
        self.when        = None
        self.last_author = None
        self.descr       = None
        self.actions     = []
        self.statehistory = []
        self.reviews      = []

    def read(self, root):
        self.reqid = int(root.get('id'))
        actions = root.findall('action')
        if len(actions) == 0:
            actions = [ root.find('submit') ] # for old style requests

        for action in actions:
            type = action.get('type', 'submit')
            try:
                src_prj = src_pkg = src_rev = dst_prj = dst_pkg = src_update = None
                if action.findall('source'):
                    n = action.find('source')
                    src_prj = n.get('project', None)
                    src_pkg = n.get('package', None)
                    src_rev = n.get('rev', None)
                if action.findall('target'):
                    n = action.find('target')
                    dst_prj = n.get('project', None)
                    dst_pkg = n.get('package', None)
                if action.findall('options'):
                    n = action.find('options')
                    if n.findall('sourceupdate'):
                        src_update = n.find('sourceupdate').text.strip()
                self.add_action(type, src_prj, src_pkg, src_rev, dst_prj, dst_pkg, src_update)
            except:
                msg = 'invalid request format:\n%s' % ET.tostring(root)
                raise oscerr.APIError(msg)

        # read the state
        n = root.find('state')
        self.state.name, self.state.who, self.state.when \
                = n.get('name'), n.get('who'), n.get('when')
        try:
            self.state.comment = n.find('comment').text.strip()
        except:
            self.state.comment = None

        # read the review states
        for r in root.findall('review'):
            s = ReviewState()
            s.state    = r.get('state')
            s.by_user  = r.get('by_user')
            s.by_group = r.get('by_group')
            s.who      = r.get('who')
            s.when     = r.get('when')
            try:
                s.comment = r.find('comment').text.strip()
            except:
                s.comment = None
            self.reviews.append(s)

        # read the state history
        for h in root.findall('history'):
            s = RequestState()
            s.name = h.get('name')
            s.who  = h.get('who')
            s.when = h.get('when')
            try:
                s.comment = h.find('comment').text.strip()
            except:
                s.comment = None
            self.statehistory.append(s)
        self.statehistory.reverse()

        # read a description, if it exists
        try:
            n = root.find('description').text
            self.descr = n
        except:
            pass

    def add_action(self, type, src_prj, src_pkg, src_rev, dst_prj, dst_pkg, src_update):
        self.actions.append(Action(type, src_prj, src_pkg, src_rev,
                                   dst_prj, dst_pkg, src_update)
                           )

    def list_view(self):
        ret = '%6d  State:%-7s By:%-12s When:%-12s' % (self.reqid, self.state.name, self.state.who, self.state.when)

        for a in self.actions:
            dst = "%s/%s" % (a.dst_project, a.dst_package)
            if a.src_package == a.dst_package:
                dst = a.dst_project

            sr_source=""
            if a.type=="submit":
                sr_source="%s/%s  -> " % (a.src_project, a.src_package)
            if a.type=="change_devel":
                dst = "developed in %s/%s" % (a.src_project, a.src_package)
                sr_source="%s/%s" % (a.dst_project, a.dst_package)

            ret += '\n        %s:       %-50s %-20s   ' % \
            (a.type, sr_source, dst)

        if self.statehistory and self.statehistory[0]:
            who = []
            for h in self.statehistory:
                who.append("%s(%s)" % (h.who,h.name))
            who.reverse()
            ret += "\n        From: %s" % (' -> '.join(who))
        if self.descr:
            txt = re.sub(r'[^[:isprint:]]', '_', self.descr)
            import textwrap
            lines = txt.splitlines()
            wrapper = textwrap.TextWrapper( width = 80,
                    initial_indent='        Descr: ',
                    subsequent_indent='               ')
            ret += "\n" + wrapper.fill(lines[0])
            wrapper.initial_indent = '               '
            for line in lines[1:]:
                ret += "\n" + wrapper.fill(line)

        ret += "\n"

        return ret

    def __cmp__(self, other):
        return cmp(self.reqid, other.reqid)

    def __str__(self):
        action_list=""
        for action in self.actions:
            action_list=action_list+"  %s:  " % (action.type)
            if action.type=="submit":
                r=""
                if action.src_rev:
                    r="(r%s)" % (action.src_rev)
                m=""
                if action.src_update:
                    m="(%s)" % (action.src_update)
                action_list=action_list+" %s/%s%s%s -> %s" % ( action.src_project, action.src_package, r, m, action.dst_project )
                if action.dst_package:
                    action_list=action_list+"/%s" % ( action.dst_package )
            elif action.type=="delete":
                action_list=action_list+"  %s" % ( action.dst_project )
                if action.dst_package:
                    action_list=action_list+"/%s" % ( action.dst_package )
            elif action.type=="change_devel":
                action_list=action_list+" %s/%s developed in %s/%s" % \
                           ( action.dst_project, action.dst_package, action.src_project, action.src_package )
            action_list=action_list+"\n"

        s = """\
Request #%s:

%s

Message:
    %s

State:   %-10s   %s %s
Comment: %s
"""          % (self.reqid,
               action_list,
               self.descr,
               self.state.name, self.state.when, self.state.who,
               self.state.comment)

        if len(self.reviews):
            reviewitems = [ '%-10s  %s %s %s %s   %s' \
                    % (i.state, i.by_user, i.by_group, i.when, i.who, i.comment) \
                    for i in self.reviews ]
            s += '\nReview:  ' + '\n         '.join(reviewitems)

        s += '\n'
        if len(self.statehistory):
            histitems = [ '%-10s   %s %s' \
                    % (i.name, i.when, i.who) \
                    for i in self.statehistory ]
            s += '\nHistory: ' + '\n         '.join(histitems)

        s += '\n'
        return s


def shorttime(t):
    """format time as Apr 02 18:19
    or                Apr 02  2005
    depending on whether it is in the current year
    """
    import time

    if time.localtime()[0] == time.localtime(t)[0]:
        # same year
        return time.strftime('%b %d %H:%M',time.localtime(t))
    else:
        return time.strftime('%b %d  %Y',time.localtime(t))


def is_project_dir(d):
    return os.path.exists(os.path.join(d, store, '_project')) and not \
           os.path.exists(os.path.join(d, store, '_package'))


def is_package_dir(d):
    return os.path.exists(os.path.join(d, store, '_project')) and \
           os.path.exists(os.path.join(d, store, '_package'))

def parse_disturl(disturl):
    """Parse a disturl, returns tuple (apiurl, project, source, repository,
    revision), else raises an oscerr.WrongArgs exception
    """

    m = DISTURL_RE.match(disturl)
    if not m:
        raise oscerr.WrongArgs("`%s' does not look like disturl" % disturl)

    apiurl = m.group('apiurl')
    if apiurl.split('.')[0] != 'api':
        apiurl = 'https://api.' + ".".join(apiurl.split('.')[1:])
    return (apiurl, m.group('project'), m.group('source'), m.group('repository'), m.group('revision'))

def parse_buildlogurl(buildlogurl):
    """Parse a build log url, returns a tuple (apiurl, project, package,
    repository, arch), else raises oscerr.WrongArgs exception"""

    global BUILDLOGURL_RE

    m = BUILDLOGURL_RE.match(buildlogurl)
    if not m:
        raise oscerr.WrongArgs('\'%s\' does not look like url with a build log' % buildlogurl)

    return (m.group('apiurl'), m.group('project'), m.group('package'), m.group('repository'), m.group('arch'))

def slash_split(l):
    """Split command line arguments like 'foo/bar' into 'foo' 'bar'.
    This is handy to allow copy/paste a project/package combination in this form.

    Trailing slashes are removed before the split, because the split would
    otherwise give an additional empty string.
    """
    r = []
    for i in l:
        i = i.rstrip('/')
        r += i.split('/')
    return r

def expand_proj_pack(args, idx=0, howmany=0):
    """looks for occurance of '.' at the position idx.
    If howmany is 2, both proj and pack are expanded together
    using the current directory, or none of them, if not possible.
    If howmany is 0, proj is expanded if possible, then, if there
    is no idx+1 element in args (or args[idx+1] == '.'), pack is also
    expanded, if possible.
    If howmany is 1, only proj is expanded if possible.

    If args[idx] does not exists, an implicit '.' is assumed.
    if not enough elements up to idx exist, an error is raised.

    See also parseargs(args), slash_split(args), findpacs(args)
    All these need unification, somehow.
    """

    # print args,idx,howmany

    if len(args) < idx:
        raise oscerr.WrongArgs('not enough argument, expected at least %d' % idx)

    if len(args) == idx:
        args += '.'
    if args[idx+0] == '.':
        if howmany == 0 and len(args) > idx+1:
            if args[idx+1] == '.':
                # we have two dots.
                # remove one dot and make sure to expand both proj and pack
                args.pop(idx+1)
                howmany = 2
            else:
                howmany = 1
        # print args,idx,howmany

        args[idx+0] = store_read_project('.')
        if howmany == 0:
            try:
                package = store_read_package('.')
                args.insert(idx+1, package)
            except:
                pass
        elif howmany == 2:
            package = store_read_package('.')
            args.insert(idx+1, package)
    return args


def findpacs(files, progress_obj=None):
    """collect Package objects belonging to the given files
    and make sure each Package is returned only once"""
    pacs = []
    for f in files:
        p = filedir_to_pac(f, progress_obj)
        known = None
        for i in pacs:
            if i.name == p.name:
                known = i
                break
        if known:
            i.merge(p)
        else:
            pacs.append(p)
    return pacs


def filedir_to_pac(f, progress_obj=None):
    """Takes a working copy path, or a path to a file inside a working copy,
    and returns a Package object instance

    If the argument was a filename, add it onto the "todo" list of the Package """

    if os.path.isdir(f):
        wd = f
        p = Package(wd, progress_obj=progress_obj)
    else:
        wd = os.path.dirname(f) or os.curdir
        p = Package(wd, progress_obj=progress_obj)
        p.todo = [ os.path.basename(f) ]
    return p


def read_filemeta(dir):
    try:
        r = ET.parse(os.path.join(dir, store, '_files'))
    except SyntaxError, e:
        raise oscerr.NoWorkingCopy('\'%s\' is not a valid working copy.\n'
                                   'When parsing .osc/_files, the following error was encountered:\n'
                                   '%s' % (dir, e))
    return r


def read_tobedeleted(dir):
    r = []
    fname = os.path.join(dir, store, '_to_be_deleted')

    if os.path.exists(fname):
        r = [ line.strip() for line in open(fname) ]

    return r


def read_sizelimit(dir):
    r = None
    fname = os.path.join(dir, store, '_size_limit')

    if os.path.exists(fname):
        r = open(fname).readline()

    if r is None or not r.isdigit():
        return None
    return int(r)

def read_inconflict(dir):
    r = []
    fname = os.path.join(dir, store, '_in_conflict')

    if os.path.exists(fname):
        r = [ line.strip() for line in open(fname) ]

    return r


def parseargs(list_of_args):
    """Convenience method osc's commandline argument parsing.

    If called with an empty tuple (or list), return a list containing the current directory.
    Otherwise, return a list of the arguments."""
    if list_of_args:
        return list(list_of_args)
    else:
        return [os.curdir]


def statfrmt(statusletter, filename):
    return '%s    %s' % (statusletter, filename)


def pathjoin(a, *p):
    """Join two or more pathname components, inserting '/' as needed. Cut leading ./"""
    path = os.path.join(a, *p)
    if path.startswith('./'):
        path = path[2:]
    return path


def makeurl(baseurl, l, query=[]):
    """Given a list of path compoments, construct a complete URL.

    Optional parameters for a query string can be given as a list, as a
    dictionary, or as an already assembled string.
    In case of a dictionary, the parameters will be urlencoded by this
    function. In case of a list not -- this is to be backwards compatible.
    """

    if conf.config['verbose'] > 1:
        print 'makeurl:', baseurl, l, query

    if type(query) == type(list()):
        query = '&'.join(query)
    elif type(query) == type(dict()):
        query = urlencode(query)

    scheme, netloc = urlsplit(baseurl)[0:2]
    return urlunsplit((scheme, netloc, '/'.join(l), query, ''))


def http_request(method, url, headers={}, data=None, file=None, timeout=100):
    """wrapper around urllib2.urlopen for error handling,
    and to support additional (PUT, DELETE) methods"""

    filefd = None

    if conf.config['http_debug']:
        print
        print
        print '--', method, url

    if method == 'POST' and not file and not data:
        # adding data to an urllib2 request transforms it into a POST
        data = ''

    req = urllib2.Request(url)
    api_host_options = {}
    try:
        api_host_options = conf.get_apiurl_api_host_options(url)
        for header, value in api_host_options['http_headers']:
            req.add_header(header, value)
    except:
        # "external" request (url is no apiurl)
        pass

    req.get_method = lambda: method

    # POST requests are application/x-www-form-urlencoded per default
    # since we change the request into PUT, we also need to adjust the content type header
    if method == 'PUT' or (method == 'POST' and data):
        req.add_header('Content-Type', 'application/octet-stream')

    if type(headers) == type({}):
        for i in headers.keys():
            print headers[i]
            req.add_header(i, headers[i])

    if file and not data:
        size = os.path.getsize(file)
        if size < 1024*512:
            data = open(file, 'rb').read()
        else:
            import mmap
            filefd = open(file, 'rb')
            try:
                if sys.platform[:3] != 'win':
                    data = mmap.mmap(filefd.fileno(), os.path.getsize(file), mmap.MAP_SHARED, mmap.PROT_READ)
                else:
                    data = mmap.mmap(filefd.fileno(), os.path.getsize(file))
                data = buffer(data)
            except EnvironmentError, e:
                if e.errno == 19:
                    sys.exit('\n\n%s\nThe file \'%s\' could not be memory mapped. It is ' \
                             '\non a filesystem which does not support this.' % (e, file))
                elif hasattr(e, 'winerror') and e.winerror == 5:
                    # falling back to the default io
                    data = open(file, 'rb').read()
                else:
                    raise

    if conf.config['debug']: print method, url

    old_timeout = socket.getdefaulttimeout()
    # XXX: dirty hack as timeout doesn't work with python-m2crypto
    if old_timeout != timeout and not api_host_options.get('sslcertck'):
        socket.setdefaulttimeout(timeout)
    try:
        fd = urllib2.urlopen(req, data=data)
    finally:
        if old_timeout != timeout and not api_host_options.get('sslcertck'):
            socket.setdefaulttimeout(old_timeout)
        if hasattr(conf.cookiejar, 'save'):
            conf.cookiejar.save(ignore_discard=True)

    if filefd: filefd.close()

    return fd


def http_GET(*args, **kwargs):    return http_request('GET', *args, **kwargs)
def http_POST(*args, **kwargs):   return http_request('POST', *args, **kwargs)
def http_PUT(*args, **kwargs):    return http_request('PUT', *args, **kwargs)
def http_DELETE(*args, **kwargs): return http_request('DELETE', *args, **kwargs)


def init_project_dir(apiurl, dir, project):
    if not os.path.exists(dir):
        if conf.config['checkout_no_colon']:
            os.makedirs(dir)      # helpful with checkout_no_colon
        else:
            os.mkdir(dir)
    if not os.path.exists(os.path.join(dir, store)):
        os.mkdir(os.path.join(dir, store))

    # print 'project=',project,'  dir=',dir
    store_write_project(dir, project)
    store_write_apiurl(dir, apiurl)
    if conf.config['do_package_tracking']:
        store_write_initial_packages(dir, project, [])

def init_package_dir(apiurl, project, package, dir, revision=None, files=True, limit_size=None, meta=False):
    if not os.path.isdir(dir):
        os.mkdir(dir)
    os.chdir(dir)
    f = open('_project', 'w')
    f.write(project + '\n')
    f.close()
    f = open('_package', 'w')
    f.write(package + '\n')
    f.close()

    if meta:
        store_write_string(os.pardir, '_meta_mode', '')

    if limit_size:
        store_write_string(os.pardir, '_size_limit', str(limit_size))

    if files:
        fmeta = ''.join(show_files_meta(apiurl, project, package, revision=revision, limit_size=limit_size, meta=meta))
        store_write_string(os.pardir, '_files', fmeta)
    else:
        # create dummy
        ET.ElementTree(element=ET.Element('directory')).write('_files')

    store_write_string(os.pardir, '_osclib_version', __store_version__ + '\n')
    store_write_apiurl(os.path.pardir, apiurl)
    os.chdir(os.pardir)


def check_store_version(dir):
    versionfile = os.path.join(dir, store, '_osclib_version')
    try:
        v = open(versionfile).read().strip()
    except:
        v = ''

    if v == '':
        msg = 'Error: "%s" is not an osc package working copy.' % os.path.abspath(dir)
        if os.path.exists(os.path.join(dir, '.svn')):
            msg = msg + '\nTry svn instead of osc.'
        raise oscerr.NoWorkingCopy(msg)

    if v != __store_version__:
        if v in ['0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9', '0.95', '0.96', '0.97', '0.98', '0.99']:
            # version is fine, no migration needed
            f = open(versionfile, 'w')
            f.write(__store_version__ + '\n')
            f.close()
            return
        msg = 'The osc metadata of your working copy "%s"' % dir
        msg += '\nhas __store_version__ = %s, but it should be %s' % (v, __store_version__)
        msg += '\nPlease do a fresh checkout or update your client. Sorry about the inconvenience.'
        raise oscerr.WorkingCopyWrongVersion, msg


def meta_get_packagelist(apiurl, prj, deleted=None):

    query = {}
    if deleted:
       query['deleted'] = 1

    u = makeurl(apiurl, ['source', prj], query)
    f = http_GET(u)
    root = ET.parse(f).getroot()
    return [ node.get('name') for node in root.findall('entry') ]


def meta_get_filelist(apiurl, prj, package, verbose=False, expand=False, revision=None):
    """return a list of file names,
    or a list File() instances if verbose=True"""

    query = {}
    if expand:
        query['expand'] = 1
    if revision:
        query['rev'] = revision
    else:
        query['rev'] = 'latest'

    u = makeurl(apiurl, ['source', prj, package], query=query)
    f = http_GET(u)
    root = ET.parse(f).getroot()

    if not verbose:
        return [ node.get('name') for node in root.findall('entry') ]

    else:
        l = []
        # rev = int(root.get('rev'))    # don't force int. also allow srcmd5 here.
        rev = root.get('rev')
        for node in root.findall('entry'):
            f = File(node.get('name'),
                     node.get('md5'),
                     int(node.get('size')),
                     int(node.get('mtime')))
            f.rev = rev
            l.append(f)
        return l


def meta_get_project_list(apiurl, deleted=None):
    query = {}
    if deleted:
        query['deleted'] = 1

    u = makeurl(apiurl, ['source'], query)
    f = http_GET(u)
    root = ET.parse(f).getroot()
    return sorted([ node.get('name') for node in root ])


def show_project_meta(apiurl, prj):
    url = makeurl(apiurl, ['source', prj, '_meta'])
    f = http_GET(url)
    return f.readlines()


def show_project_conf(apiurl, prj):
    url = makeurl(apiurl, ['source', prj, '_config'])
    f = http_GET(url)
    return f.readlines()


def show_package_trigger_reason(apiurl, prj, pac, repo, arch):
    url = makeurl(apiurl, ['build', prj, repo, arch, pac, '_reason'])
    try:
        f = http_GET(url)
        return f.read()
    except urllib2.HTTPError, e:
        e.osc_msg = 'Error getting trigger reason for project \'%s\' package \'%s\'' % (prj, pac)
        raise


def show_package_meta(apiurl, prj, pac, meta=False):
    query = {}
    if meta:
        query['meta'] = 1

    # packages like _pattern and _project do not have a _meta file
    if pac.startswith('_pattern') or pac.startswith('_project'):
        return ""

    url = makeurl(apiurl, ['source', prj, pac, '_meta'], query)
    try:
        f = http_GET(url)
        return f.readlines()
    except urllib2.HTTPError, e:
        e.osc_msg = 'Error getting meta for project \'%s\' package \'%s\'' % (prj, pac)
        raise


def show_attribute_meta(apiurl, prj, pac, subpac, attribute, with_defaults, with_project):
    path=[]
    path.append('source')
    path.append(prj)
    if pac:
        path.append(pac)
    if pac and subpac:
        path.append(subpac)
    path.append('_attribute')
    if attribute:
        path.append(attribute)
    query=[]
    if with_defaults:
        query.append("with_default=1")
    if with_project:
        query.append("with_project=1")
    url = makeurl(apiurl, path, query)
    try:
        f = http_GET(url)
        return f.readlines()
    except urllib2.HTTPError, e:
        e.osc_msg = 'Error getting meta for project \'%s\' package \'%s\'' % (prj, pac)
        raise


def show_develproject(apiurl, prj, pac):
    m = show_package_meta(apiurl, prj, pac)
    try:
        return ET.fromstring(''.join(m)).find('devel').get('project')
    except:
        return None


def show_package_disabled_repos(apiurl, prj, pac):
    m = show_package_meta(apiurl, prj, pac)
    try:
        root = ET.fromstring(''.join(m))
        elm = root.find('build')
        r = [ node.get('repository') for node in elm.findall('disable')]
        return r
    except:
        return None


def show_pattern_metalist(apiurl, prj):
    url = makeurl(apiurl, ['source', prj, '_pattern'])
    try:
        f = http_GET(url)
        tree = ET.parse(f)
    except urllib2.HTTPError, e:
        e.osc_msg = 'show_pattern_metalist: Error getting pattern list for project \'%s\'' % prj
        raise
    r = [ node.get('name') for node in tree.getroot() ]
    r.sort()
    return r


def show_pattern_meta(apiurl, prj, pattern):
    url = makeurl(apiurl, ['source', prj, '_pattern', pattern])
    try:
        f = http_GET(url)
        return f.readlines()
    except urllib2.HTTPError, e:
        e.osc_msg = 'show_pattern_meta: Error getting pattern \'%s\' for project \'%s\'' % (pattern, prj)
        raise


class metafile:
    """metafile that can be manipulated and is stored back after manipulation."""
    def __init__(self, url, input, change_is_required=False, file_ext='.xml'):
        import tempfile

        self.url = url
        self.change_is_required = change_is_required
        (fd, self.filename) = tempfile.mkstemp(prefix = 'osc_metafile.', suffix = file_ext)
        f = os.fdopen(fd, 'w')
        f.write(''.join(input))
        f.close()
        self.hash_orig = dgst(self.filename)

    def sync(self):
        hash = dgst(self.filename)
        if self.change_is_required and hash == self.hash_orig:
            print 'File unchanged. Not saving.'
            os.unlink(self.filename)
            return

        print 'Sending meta data...'
        # don't do any exception handling... it's up to the caller what to do in case
        # of an exception
        http_PUT(self.url, file=self.filename)
        os.unlink(self.filename)
        print 'Done.'

    def edit(self):
        try:
            while 1:
                run_editor(self.filename)
                try:
                    self.sync()
                    break
                except urllib2.HTTPError, e:
                    error_help = "%d" % e.code
                    if e.headers.get('X-Opensuse-Errorcode'):
                        error_help = "%s (%d)" % (e.headers.get('X-Opensuse-Errorcode'), e.code)

                    print >>sys.stderr, 'BuildService API error:', error_help
                    # examine the error - we can't raise an exception because we might want
                    # to try again
                    data = e.read()
                    if '<summary>' in data:
                        print >>sys.stderr, data.split('<summary>')[1].split('</summary>')[0]
                    input = raw_input('Try again? ([y/N]): ')
                    if input not in ['y', 'Y']:
                        break
        finally:
            self.discard()

    def discard(self):
        if os.path.exists(self.filename):
            print 'discarding %s' % self.filename
            os.unlink(self.filename)


# different types of metadata
metatypes = { 'prj':     { 'path': 'source/%s/_meta',
                           'template': new_project_templ,
                           'file_ext': '.xml'
                         },
              'pkg':     { 'path'     : 'source/%s/%s/_meta',
                           'template': new_package_templ,
                           'file_ext': '.xml'
                         },
              'attribute':     { 'path'     : 'source/%s/%s/_meta',
                           'template': new_attribute_templ,
                           'file_ext': '.xml'
                         },
              'prjconf': { 'path': 'source/%s/_config',
                           'template': '',
                           'file_ext': '.txt'
                         },
              'user':    { 'path': 'person/%s',
                           'template': new_user_template,
                           'file_ext': '.xml'
                         },
              'pattern': { 'path': 'source/%s/_pattern/%s',
                           'template': new_pattern_template,
                           'file_ext': '.xml'
                         },
            }

def meta_exists(metatype,
                path_args=None,
                template_args=None,
                create_new=True,
                apiurl=None):

    if not apiurl:
        apiurl = conf.config['apiurl']
    url = make_meta_url(metatype, path_args, apiurl)
    try:
        data = http_GET(url).readlines()
    except urllib2.HTTPError, e:
        if e.code == 404 and create_new:
            data = metatypes[metatype]['template']
            if template_args:
                data = StringIO(data % template_args).readlines()
        else:
            raise e
    return data

def make_meta_url(metatype, path_args=None, apiurl=None):
    if not apiurl:
        apiurl = conf.config['apiurl']
    if metatype not in metatypes.keys():
        raise AttributeError('make_meta_url(): Unknown meta type \'%s\'' % metatype)
    path = metatypes[metatype]['path']

    if path_args:
        path = path % path_args

    return makeurl(apiurl, [path])


def edit_meta(metatype,
              path_args=None,
              data=None,
              template_args=None,
              edit=False,
              change_is_required=False,
              apiurl=None):

    if not apiurl:
        apiurl = conf.config['apiurl']
    if not data:
        data = meta_exists(metatype,
                           path_args,
                           template_args,
                           create_new = metatype != 'prjconf', # prjconf always exists, 404 => unknown prj
                           apiurl=apiurl)

    if edit:
        change_is_required = True

    url = make_meta_url(metatype, path_args, apiurl)
    f=metafile(url, data, change_is_required, metatypes[metatype]['file_ext'])

    if edit:
        f.edit()
    else:
        f.sync()


def show_files_meta(apiurl, prj, pac, revision=None, expand=False, linkrev=None, linkrepair=False, limit_size=None, meta=False):
    query = {}
    if revision:
        query['rev'] = revision
    else:
        query['rev'] = 'latest'
    if linkrev:
        query['linkrev'] = linkrev
    elif conf.config['linkcontrol']:
        query['linkrev'] = 'base'
    if meta:
        query['meta'] = 1
    if expand:
        query['expand'] = 1
    if linkrepair:
        query['emptylink'] = 1
    f = http_GET(makeurl(apiurl, ['source', prj, pac], query=query))
    # look for "too large" files according to size limit and mark them
    root = ET.fromstring(''.join(f.readlines()))
    for e in root.findall('entry'):
        size = e.get('size')
        if size and limit_size and int(size) > int(limit_size):
             e.set('skipped', 'true')
    return ET.tostring(root)


def show_upstream_srcmd5(apiurl, prj, pac, expand=False, revision=None, meta=False):
    m = show_files_meta(apiurl, prj, pac, expand=expand, revision=revision, meta=meta)
    return ET.fromstring(''.join(m)).get('srcmd5')


def show_upstream_xsrcmd5(apiurl, prj, pac, revision=None, linkrev=None, linkrepair=False, meta=False):
    m = show_files_meta(apiurl, prj, pac, revision=revision, linkrev=linkrev, linkrepair=linkrepair, meta=meta)
    try:
        # only source link packages have a <linkinfo> element.
        li_node = ET.fromstring(''.join(m)).find('linkinfo')
    except:
        return None

    li = Linkinfo()
    li.read(li_node)

    if li.haserror():
        raise oscerr.LinkExpandError(prj, pac, li.error)
    return li.xsrcmd5


def show_upstream_rev(apiurl, prj, pac, meta=False):
    m = show_files_meta(apiurl, prj, pac, meta=meta)
    return ET.fromstring(''.join(m)).get('rev')


def read_meta_from_spec(specfile, *args):
    import codecs, locale, re
    """
    Read tags and sections from spec file. To read out
    a tag the passed argument mustn't end with a colon. To
    read out a section the passed argument must start with
    a '%'.
    This method returns a dictionary which contains the
    requested data.
    """

    if not os.path.isfile(specfile):
        raise IOError('\'%s\' is not a regular file' % specfile)

    try:
        lines = codecs.open(specfile, 'r', locale.getpreferredencoding()).readlines()
    except UnicodeDecodeError:
        lines = open(specfile).readlines()

    tags = []
    sections = []
    spec_data = {}

    for itm in args:
        if itm.startswith('%'):
            sections.append(itm)
        else:
            tags.append(itm)

    tag_pat = '(?P<tag>^%s)\s*:\s*(?P<val>.*)'
    for tag in tags:
        m = re.compile(tag_pat % tag, re.I | re.M).search(''.join(lines))
        if m and m.group('val'):
            spec_data[tag] = m.group('val').strip()

    section_pat = '^%s\s*?$'
    for section in sections:
        m = re.compile(section_pat % section, re.I | re.M).search(''.join(lines))
        if m:
            start = lines.index(m.group()+'\n') + 1
        data = []
        for line in lines[start:]:
            if line.startswith('%'):
                break
            data.append(line)
        spec_data[section] = data

    return spec_data

def get_default_editor():
    import platform
    system = platform.system()
    if system == 'Windows':
        return 'notepad'
    if system == 'Linux':
        try:
            # Python 2.6
            dist = platform.linux_distribution()[0]
        except AttributeError:
            dist = platform.dist()[0]
        if dist == 'debian':
            return 'editor'
        elif dist == 'fedora':
            return 'vi'
        return 'vim'
    return 'vi'

def get_default_pager():
    import platform
    system = platform.system()
    if system == 'Windows':
        return 'less'
    if system == 'Linux':
        try:
            # Python 2.6
            dist = platform.linux_distribution()[0]
        except AttributeError:
            dist = platform.dist()[0]
        if dist == 'debian':
            return 'pager'
        return 'less'
    return 'more'

def run_pager(message):
    import tempfile, sys

    if not sys.stdout.isatty():
        print message
    else:
        tmpfile = tempfile.NamedTemporaryFile()
        tmpfile.write(message)
        tmpfile.flush()
        pager = os.getenv('PAGER', default=get_default_pager())
        try:
            try:
                subprocess.call('%s %s' % (pager, tmpfile.name), shell=True)
            except OSError, e:
                raise oscerr.RuntimeError('cannot run pager \'%s\': %s' % (pager, e.strerror), pager)
        finally:
            tmpfile.close()

def run_editor(filename):
    editor = os.getenv('EDITOR', default=get_default_editor())
    cmd = editor.split(' ')
    cmd.append(filename)
    try:
        return subprocess.call(cmd)
    except OSError, e:
        raise oscerr.RuntimeError('cannot run editor \'%s\': %s' % (editor, e.strerror), editor)

def edit_message(footer='', template='', templatelen=30):
    delim = '--This line, and those below, will be ignored--\n'
    import tempfile
    (fd, filename) = tempfile.mkstemp(prefix = 'osc-commitmsg', suffix = '.diff')
    f = os.fdopen(fd, 'w')
    if template != '':
        if not templatelen is None:
            lines = template.splitlines()
            template = '\n'.join(lines[:templatelen])
            if lines[templatelen:]:
                footer = '%s\n\n%s' % ('\n'.join(lines[templatelen:]), footer)
        f.write(template)
    f.write('\n')
    f.write(delim)
    f.write('\n')
    f.write(footer)
    f.close()

    try:
        while 1:
            run_editor(filename)
            msg = open(filename).read().split(delim)[0].rstrip()

            if len(msg):
                break
            else:
                input = raw_input('Log message not specified\n'
                                  'a)bort, c)ontinue, e)dit: ')
                if input in 'aA':
                    raise oscerr.UserAbort()
                elif input in 'cC':
                    break
                elif input in 'eE':
                    pass
    finally:
        os.unlink(filename)
    return msg


def create_delete_request(apiurl, project, package, message):

    import cgi

    if package:
        package = """package="%s" """ % (package)
    else:
        package = ""

    xml = """\
<request>
    <action type="delete">
        <target project="%s" %s/>
    </action>
    <state name="new"/>
    <description>%s</description>
</request>
""" % (project, package,
       cgi.escape(message or ''))

    u = makeurl(apiurl, ['request'], query='cmd=create')
    f = http_POST(u, data=xml)

    root = ET.parse(f).getroot()
    return root.get('id')


def create_change_devel_request(apiurl,
                                devel_project, devel_package,
                                project, package,
                                message):

    import cgi
    xml = """\
<request>
    <action type="change_devel">
        <source project="%s" package="%s" />
        <target project="%s" package="%s" />
    </action>
    <state name="new"/>
    <description>%s</description>
</request>
""" % (devel_project,
       devel_package,
       project,
       package,
       cgi.escape(message or ''))

    u = makeurl(apiurl, ['request'], query='cmd=create')
    f = http_POST(u, data=xml)

    root = ET.parse(f).getroot()
    return root.get('id')


# This creates an old style submit request for server api 1.0
def create_submit_request(apiurl,
                         src_project, src_package,
                         dst_project=None, dst_package=None,
                         message=None, orev=None, src_update=None):

    import cgi
    options_block=""
    if src_update:
        options_block="""<options><sourceupdate>%s</sourceupdate></options> """ % (src_update)

    # Yes, this kind of xml construction is horrible
    targetxml = ""
    if dst_project:
        packagexml = ""
        if dst_package:
            packagexml = """package="%s" """ %( dst_package )
        targetxml = """<target project="%s" %s /> """ %( dst_project, packagexml )
    # XXX: keep the old template for now in order to work with old obs instances
    xml = """\
<request type="submit">
    <submit>
        <source project="%s" package="%s" rev="%s"/>
        %s
        %s
    </submit>
    <state name="new"/>
    <description>%s</description>
</request>
""" % (src_project,
       src_package,
       orev or show_upstream_rev(apiurl, src_project, src_package),
       targetxml,
       options_block,
       cgi.escape(message or ""))

    u = makeurl(apiurl, ['request'], query='cmd=create')
    f = http_POST(u, data=xml)

    root = ET.parse(f).getroot()
    return root.get('id')


def get_request(apiurl, reqid):
    u = makeurl(apiurl, ['request', reqid])
    f = http_GET(u)
    root = ET.parse(f).getroot()

    r = Request()
    r.read(root)
    return r


def change_review_state(apiurl, reqid, newstate, by_user='', message='', supersed=''):
    u = makeurl(apiurl,
                ['request', reqid],
                query={'cmd': 'changereviewstate', 'newstate': newstate, 'by_user': by_user, 'superseded_by': supersed})
    f = http_POST(u, data=message)
    return f.read()

def change_request_state(apiurl, reqid, newstate, message='', supersed=''):
    u = makeurl(apiurl,
                ['request', reqid],
                query={'cmd': 'changestate', 'newstate': newstate, 'superseded_by': supersed})
    f = http_POST(u, data=message)
    return f.read()


def get_request_list(apiurl, project='', package='', req_who='', req_state=('new',), req_type=None, exclude_target_projects=[]):
    xpath = ''
    if not 'all' in req_state:
        for state in req_state:
            xpath = xpath_join(xpath, 'state/@name=\'%s\'' % state, inner=True)
    if req_who:
        xpath = xpath_join(xpath, '(state/@who=\'%(who)s\' or history/@who=\'%(who)s\')' % {'who': req_who}, op='and')

    # XXX: we cannot use the '|' in the xpath expression because it is not supported
    #      in the backend
    todo = {}
    if project:
        todo['project'] = project
    if package:
        todo['package'] = package
    for kind, val in todo.iteritems():
        xpath = xpath_join(xpath, '(action/target/@%(kind)s=\'%(val)s\' or ' \
                                  'action/source/@%(kind)s=\'%(val)s\' or ' \
                                  'submit/target/@%(kind)s=\'%(val)s\' or ' \
                                  'submit/source/@%(kind)s=\'%(val)s\')' % {'kind': kind, 'val': val}, op='and')
    if req_type:
        xpath = xpath_join(xpath, 'action/@type=\'%s\'' % req_type, op='and')
    for i in exclude_target_projects:
        xpath = xpath_join(xpath, '(not(action/target/@project=\'%(prj)s\' or ' \
                                  'submit/target/@project=\'%(prj)s\'))' % {'prj': i}, op='and')

    if conf.config['verbose'] > 1:
        print '[ %s ]' % xpath
    res = search(apiurl, request=xpath)
    collection = res['request']
    requests = []
    for root in collection.findall('request'):
        r = Request()
        r.read(root)
        requests.append(r)
    return requests

def get_user_projpkgs_request_list(apiurl, user, req_state=('new',), req_type=None, exclude_projects=[], projpkgs={}):
    """Return all new requests for all projects/packages where is user is involved"""
    if not projpkgs:
        res = get_user_projpkgs(apiurl, user, exclude_projects=exclude_projects)
        for i in res['project_id'].findall('project'):
            projpkgs[i.get('name')] = []
        for i in res['package_id'].findall('package'):
            if not i.get('project') in projpkgs.keys():
                projpkgs.setdefault(i.get('project'), []).append(i.get('name'))
    xpath = ''
    for prj, pacs in projpkgs.iteritems():
        if not len(pacs):
            xpath = xpath_join(xpath, 'action/target/@project=\'%s\'' % prj, inner=True)
        else:
            xp = ''
            for p in pacs:
                xp = xpath_join(xp, 'action/target/@package=\'%s\'' % p, inner=True)
            xp = xpath_join(xp, 'action/target/@project=\'%s\'' % prj, op='and')
            xpath = xpath_join(xpath, xp, inner=True)
    if req_type:
        xpath = xpath_join(xpath, 'action/@type=\'%s\'' % req_type, op='and')
    if not 'all' in req_state:
        xp = ''
        for state in req_state:
            xp = xpath_join(xp, 'state/@name=\'%s\'' % state, inner=True)
        xpath = xpath_join(xp, '(%s)' % xpath, op='and')
    res = search(apiurl, request=xpath)
    result = []
    for root in res['request'].findall('request'):
        r = Request()
        r.read(root)
        result.append(r)
    return result

def get_request_log(apiurl, reqid):
    r = get_request(apiurl, reqid)
    data = []
    frmt = '-' * 76 + '\n%s | %s | %s\n\n%s'
    # the description of the request is used for the initial log entry
    # otherwise its comment attribute would contain None
    if len(r.statehistory) >= 1:
        r.statehistory[-1].comment = r.descr
    else:
        r.state.comment = r.descr
    for state in [ r.state ] + r.statehistory:
        s = frmt % (state.name, state.who, state.when, str(state.comment))
        data.append(s)
    return data


def get_user_meta(apiurl, user):
    u = makeurl(apiurl, ['person', quote_plus(user)])
    try:
        f = http_GET(u)
        return ''.join(f.readlines())
    except urllib2.HTTPError:
        print 'user \'%s\' not found' % user
        return None


def get_user_data(apiurl, user, *tags):
    """get specified tags from the user meta"""
    meta = get_user_meta(apiurl, user)
    data = []
    if meta != None:
        root = ET.fromstring(meta)
        for tag in tags:
            try:
                if root.find(tag).text != None:
                    data.append(root.find(tag).text)
                else:
                    # tag is empty
                    data.append('-')
            except AttributeError:
                # this part is reached if the tags tuple contains an invalid tag
                print 'The xml file for user \'%s\' seems to be broken' % user
                return []
    return data


def download(url, filename, progress_obj = None, mtime = None):
    import tempfile, shutil
    o = None
    try:
        prefix = os.path.basename(filename)
        (fd, tmpfile) = tempfile.mkstemp(prefix = prefix, suffix = '.osc')
        os.chmod(tmpfile, 0644)
        try:
            o = os.fdopen(fd, 'wb')
            for buf in streamfile(url, http_GET, BUFSIZE, progress_obj=progress_obj):
                o.write(buf)
            o.close()
            shutil.move(tmpfile, filename)
        except:
            os.unlink(tmpfile)