Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- 

# 

# Copyright (C) 2015 Canonical Ltd 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License version 3 as 

# published by the Free Software Foundation. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program.  If not, see <http://www.gnu.org/licenses/>. 

 

import apt 

import glob 

import itertools 

import logging 

import os 

import platform 

import string 

import shutil 

import stat 

import subprocess 

import urllib 

import urllib.request 

 

from xml.etree import ElementTree 

 

import snapcraft.common 

 

logger = logging.getLogger(__name__) 

 

_DEFAULT_SOURCES = \ 

    '''deb http://${prefix}.ubuntu.com/${suffix}/ ${release} main restricted 

deb http://${prefix}.ubuntu.com/${suffix}/ ${release}-updates main restricted 

deb http://${prefix}.ubuntu.com/${suffix}/ ${release} universe 

deb http://${prefix}.ubuntu.com/${suffix}/ ${release}-updates universe 

deb http://${prefix}.ubuntu.com/${suffix}/ ${release} multiverse 

deb http://${prefix}.ubuntu.com/${suffix}/ ${release}-updates multiverse 

deb http://${security}.ubuntu.com/${suffix} ${release}-security main restricted 

deb http://${security}.ubuntu.com/${suffix} ${release}-security universe 

deb http://${security}.ubuntu.com/${suffix} ${release}-security multiverse 

''' 

_GEOIP_SERVER = "http://geoip.ubuntu.com/lookup" 

 

 

class PackageNotFoundError(Exception): 

 

    @property 

    def message(self): 

        return 'The Ubuntu package "{}" was not found'.format( 

            self.package_name) 

 

    def __init__(self, package_name): 

        self.package_name = package_name 

 

 

class UnpackError(Exception): 

 

    @property 

    def message(self): 

        return 'Error while provisioning "{}"'.format(self.package_name) 

 

    def __init__(self, package_name): 

        self.package_name = package_name 

 

 

class Ubuntu: 

 

    def __init__(self, rootdir, recommends=False, sources=_DEFAULT_SOURCES): 

        self.downloaddir = os.path.join(rootdir, 'download') 

        self.rootdir = rootdir 

        self.recommends = recommends 

        sources = sources or _DEFAULT_SOURCES 

        local = False 

 

        if 'SNAPCRAFT_LOCAL_SOURCES' in os.environ: 

            print('using local sources') 

            sources = _get_local_sources_list() 

            local = True 

        self.apt_cache, self.apt_progress = _setup_apt_cache( 

            rootdir, sources, local) 

 

    def get(self, package_names): 

        os.makedirs(self.downloaddir, exist_ok=True) 

 

        manifest_dep_names = self._manifest_dep_names() 

 

        for name in package_names: 

            try: 

                self.apt_cache[name].mark_install() 

            except KeyError: 

                raise PackageNotFoundError(name) 

 

        skipped_essential = [] 

        skipped_blacklisted = [] 

 

        # unmark some base packages here 

        # note that this will break the consistency check inside apt_cache 

        # (self.apt_cache.broken_count will be > 0) 

        # but that is ok as it was consistent before we excluded 

        # these base package 

        for pkg in self.apt_cache: 

            # those should be already on each system, it also prevents 

            # diving into downloading libc6 

            if (pkg.candidate.priority in 'essential' and 

               pkg.name not in package_names): 

                skipped_essential.append(pkg.name) 

                pkg.mark_keep() 

                continue 

            if (pkg.name in manifest_dep_names and 

                    pkg.name not in package_names): 

                skipped_blacklisted.append(pkg.name) 

                pkg.mark_keep() 

                continue 

 

        if skipped_essential: 

            print('Skipping priority essential packages:', skipped_essential) 

        if skipped_blacklisted: 

            print('Skipping blacklisted from manifest packages:', 

                  skipped_blacklisted) 

 

        # download the remaining ones with proper progress 

        apt.apt_pkg.config.set("Dir::Cache::Archives", self.downloaddir) 

        self.apt_cache.fetch_archives(progress=self.apt_progress) 

 

    def unpack(self, rootdir): 

        pkgs_abs_path = glob.glob(os.path.join(self.downloaddir, '*.deb')) 

        for pkg in pkgs_abs_path: 

            # TODO needs elegance and error control 

            try: 

                subprocess.check_call(['dpkg-deb', '--extract', pkg, rootdir]) 

            except subprocess.CalledProcessError: 

                raise UnpackError(pkg) 

 

        _fix_contents(rootdir) 

        _fix_xml_tools(rootdir) 

 

    def _manifest_dep_names(self): 

        manifest_dep_names = set() 

 

        with open(os.path.abspath(os.path.join(__file__, '..', 

                                               'manifest.txt'))) as f: 

            for line in f: 

                pkg = line.strip() 

                if pkg in self.apt_cache: 

                    manifest_dep_names.add(pkg) 

 

        return manifest_dep_names 

 

 

def _get_local_sources_list(): 

    sources_list = glob.glob('/etc/apt/sources.list.d/*.list') 

    sources_list.append('/etc/apt/sources.list') 

 

    sources = '' 

    for source in sources_list: 

        with open(source) as f: 

            sources += f.read() 

 

    return sources 

 

 

def _get_geoip_country_code_prefix(): 

    try: 

        with urllib.request.urlopen(_GEOIP_SERVER) as f: 

            xml_data = f.read() 

        et = ElementTree.fromstring(xml_data) 

        cc = et.find("CountryCode") 

        if cc is None: 

            return "" 

        return cc.text.lower() 

    except (ElementTree.ParseError, urllib.error.URLError): 

        pass 

    return '' 

 

 

def _format_sources_list(sources, arch, release='vivid'): 

    if arch in ('amd64', 'i386'): 

        geoip_prefix = _get_geoip_country_code_prefix() 

        prefix = geoip_prefix + '.archive' if geoip_prefix else 'archive' 

        suffix = 'ubuntu' 

        security = 'security' 

    else: 

        prefix = 'ports' 

        suffix = 'ubuntu-ports' 

        security = 'ports' 

 

    return string.Template(sources).substitute({ 

        'prefix': prefix, 

        'release': release, 

        'suffix': suffix, 

        'security': security, 

    }) 

 

 

def _setup_apt_cache(rootdir, sources, local=False): 

    os.makedirs(os.path.join(rootdir, 'etc', 'apt'), exist_ok=True) 

    srcfile = os.path.join(rootdir, 'etc', 'apt', 'sources.list') 

 

    if not local: 

        arch = snapcraft.common.get_arch() 

        series = platform.linux_distribution()[2] 

        sources = _format_sources_list(sources, arch, series) 

 

    with open(srcfile, 'w') as f: 

        f.write(sources) 

 

    # Do not install recommends 

    apt.apt_pkg.config.set('Apt::Install-Recommends', 'False') 

 

    # Make sure we always use the system GPG configuration, even with 

    # apt.Cache(rootdir). 

    for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts': 

        apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key)) 

 

    progress = apt.progress.text.AcquireProgress() 

    if not os.isatty(1): 

        # Make output more suitable for logging. 

        progress.pulse = lambda owner: True 

        progress._width = 0 

 

    apt_cache = apt.Cache(rootdir=rootdir, memonly=True) 

    apt_cache.update(fetch_progress=progress, sources_list=srcfile) 

    apt_cache.open() 

 

    return apt_cache, progress 

 

 

def _fix_contents(debdir): 

    ''' 

    Sometimes debs will contain absolute symlinks (e.g. if the relative 

    path would go all the way to root, they just do absolute).  We can't 

    have that, so instead clean those absolute symlinks. 

 

    Some unpacked items will also contain suid binaries which we do not want in 

    the resulting snap. 

    ''' 

    for root, dirs, files in os.walk(debdir): 

        # Symlinks to directories will be in dirs, while symlinks to 

        # non-directories will be in files. 

        for entry in itertools.chain(files, dirs): 

            path = os.path.join(root, entry) 

            if os.path.islink(path) and os.path.isabs(os.readlink(path)): 

                target = os.path.join(debdir, os.readlink(path)[1:]) 

250                if _skip_link(os.readlink(path)): 

                    logger.debug('Skipping {}'.format(target)) 

                    continue 

                if not os.path.exists(target): 

255                    if not _try_copy_local(path, target): 

                        continue 

                os.remove(path) 

                os.symlink(os.path.relpath(target, root), path) 

245            elif os.path.exists(path): 

                _fix_filemode(path) 

 

 

def _fix_xml_tools(root): 

    xml2_config_path = os.path.join(root, 'usr', 'bin', 'xml2-config') 

    if os.path.isfile(xml2_config_path): 

        snapcraft.common.run( 

            ['sed', '-i', '-e', 's|prefix=/usr|prefix={}/usr|'. 

                format(root), xml2_config_path]) 

 

    xslt_config_path = os.path.join(root, 'usr', 'bin', 'xslt-config') 

    if os.path.isfile(xslt_config_path): 

        snapcraft.common.run( 

            ['sed', '-i', '-e', 's|prefix=/usr|prefix={}/usr|'. 

                format(root), xslt_config_path]) 

 

 

def _fix_filemode(path): 

    mode = stat.S_IMODE(os.stat(path, follow_symlinks=False).st_mode) 

    if mode & 0o4000 or mode & 0o2000: 

        logger.warning('Removing suid/guid from {}'.format(path)) 

        os.chmod(path, mode & 0o1777) 

 

 

_skip_list = None 

 

 

def _skip_link(target): 

    global _skip_list 

    if not _skip_list: 

        output = snapcraft.common.run_output(['dpkg', '-L', 'libc6']).split() 

        _skip_list = [i for i in output if 'lib' in i] 

 

    return target in _skip_list 

 

 

def _try_copy_local(path, target): 

    real_path = os.path.realpath(path) 

297    if os.path.exists(real_path): 

        logger.warning( 

            'Copying needed target link from the system {}'.format(real_path)) 

        os.makedirs(os.path.dirname(target), exist_ok=True) 

        shutil.copyfile(os.readlink(path), target) 

        return True 

    else: 

        logger.warning( 

            '{} will be a dangling symlink'.format(path)) 

        return False