kiwi.storage.disk

  1# Copyright (c) 2015 SUSE Linux GmbH.  All rights reserved.
  2#
  3# This file is part of kiwi.
  4#
  5# kiwi is free software: you can redistribute it and/or modify
  6# it under the terms of the GNU General Public License as published by
  7# the Free Software Foundation, either version 3 of the License, or
  8# (at your option) any later version.
  9#
 10# kiwi is distributed in the hope that it will be useful,
 11# but WITHOUT ANY WARRANTY; without even the implied warranty of
 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13# GNU General Public License for more details.
 14#
 15# You should have received a copy of the GNU General Public License
 16# along with kiwi.  If not, see <http://www.gnu.org/licenses/>
 17#
 18import os
 19import logging
 20from collections import OrderedDict
 21from typing import (
 22    Dict, NamedTuple, Tuple, Optional
 23)
 24
 25# project
 26from kiwi.defaults import Defaults
 27from kiwi.utils.temporary import Temporary
 28from kiwi.command import Command
 29from kiwi.storage.device_provider import DeviceProvider
 30from kiwi.storage.mapped_device import MappedDevice
 31from kiwi.partitioner import Partitioner
 32from kiwi.runtime_config import RuntimeConfig
 33from kiwi.exceptions import (
 34    KiwiCustomPartitionConflictError,
 35    KiwiError
 36)
 37
 38
 39class ptable_entry_type(NamedTuple):
 40    mbsize: int
 41    clone: int
 42    partition_name: str
 43    partition_type: str
 44    partition_id: Optional[int]
 45    mountpoint: str
 46    filesystem: str
 47    label: str
 48
 49
 50log = logging.getLogger('kiwi')
 51
 52
 53class Disk(DeviceProvider):
 54    """
 55    **Implements storage disk and partition table setup**
 56    """
 57    def __init__(
 58        self, table_type: str, storage_provider: DeviceProvider,
 59        start_sector: int = None, extended_layout: bool = False
 60    ):
 61        """
 62        Construct a new Disk layout object
 63
 64        :param string table_type: Partition table type name
 65        :param object storage_provider:
 66            Instance of class based on DeviceProvider
 67        :param int start_sector: sector number
 68        :param bool extended_layout:
 69            If set to true and on msdos table type when creating
 70            more than 4 partitions, this will cause the fourth
 71            partition to be an extended partition and all following
 72            partitions will be placed as logical partitions inside
 73            of that extended partition
 74        """
 75        self.partition_mapper = RuntimeConfig().get_mapper_tool()
 76        #: the underlaying device provider
 77        self.storage_provider = storage_provider
 78
 79        #: list of protected map ids. If used in a custom partitions
 80        #: setup this will lead to a raise conditition in order to
 81        #: avoid conflicts with the existing partition layout and its
 82        #: customizaton capabilities
 83        self.protected_map_ids = [
 84            'root',
 85            'readonly',
 86            'boot',
 87            'prep',
 88            'spare',
 89            'swap',
 90            'efi_csm',
 91            'efi'
 92        ]
 93
 94        #: Unified partition UUIDs according to systemd
 95        self.gUID = self.get_discoverable_partition_ids()
 96
 97        self.partition_map: Dict[str, str] = {}
 98        self.public_partition_id_map: Dict[str, str] = {}
 99        self.partition_id_map: Dict[str, str] = {}
100        self.is_mapped = False
101
102        self.partitioner = Partitioner.new(
103            table_type, storage_provider, start_sector, extended_layout
104        )
105
106        self.table_type = table_type
107
108    def __enter__(self):
109        return self
110
111    def get_device(self) -> Dict[str, MappedDevice]:
112        """
113        Names of partition devices
114
115        Note that the mapping requires an explicit map() call
116
117        :return: instances of MappedDevice
118
119        :rtype: dict
120        """
121        device_map = {}
122        for partition_name, device_node in list(self.partition_map.items()):
123            device_map[partition_name] = MappedDevice(
124                device=device_node, device_provider=self
125            )
126        return device_map
127
128    def is_loop(self) -> bool:
129        """
130        Check if storage provider is loop based
131
132        The information is taken from the storage provider. If
133        the storage provider is loop based the disk is it too
134
135        :return: True or False
136
137        :rtype: bool
138        """
139        return self.storage_provider.is_loop()
140
141    def create_custom_partitions(
142        self, table_entries: Dict[str, ptable_entry_type]
143    ) -> None:
144        """
145        Create partitions from custom data set
146
147        .. code:: python
148
149           table_entries = {
150               map_name: ptable_entry_type
151           }
152
153        :param dict table: partition table spec
154        """
155        for map_name in table_entries:
156            if map_name in self.protected_map_ids:
157                raise KiwiCustomPartitionConflictError(
158                    f'Cannot use reserved table entry name: {map_name!r}'
159                )
160            entry = table_entries[map_name]
161            if entry.clone:
162                self._create_clones(
163                    map_name, entry.clone, entry.partition_type,
164                    format(entry.mbsize), entry.partition_id
165                )
166            id_name = f'kiwi_{map_name.title()}Part'
167            self.partitioner.create(
168                name=entry.partition_name,
169                mbsize=entry.mbsize,
170                type_name=entry.partition_type,
171                partition_id=entry.partition_id
172            )
173            self._add_to_map(map_name)
174            self._add_to_public_id_map(id_name)
175            part_uuid = self.gUID.get(entry.partition_name)
176            if part_uuid:
177                self.partitioner.set_uuid(
178                    self.partition_id_map[map_name], part_uuid
179                )
180
181    def create_root_partition(
182        self, mbsize: str, clone: int = 0, partition_id: Optional[int] = None
183    ):
184        """
185        Create root partition
186
187        Populates kiwi_RootPart(id) and kiwi_BootPart(id) if no extra
188        boot partition is requested
189
190        :param str mbsize: partition size string
191        :param int clone: create [clone] cop(y/ies) of the root partition
192        :param int partition_id:
193            If provided, use this exact partition ID
194            instead of auto-incrementing. When cloned, the clone
195            ID is calculated from the given partition_id
196        """
197        (mbsize, mbsize_clone) = Disk._parse_size(mbsize)
198        if clone:
199            self._create_clones(
200                'root', clone, 't.linux', mbsize_clone, partition_id
201            )
202        self.partitioner.create(
203            name='p.lxroot',
204            mbsize=mbsize,
205            type_name='t.linux',
206            partition_id=partition_id
207        )
208        self._add_to_map('root')
209        self._add_to_public_id_map('kiwi_RootPart')
210        if 'kiwi_ROPart' in self.public_partition_id_map:
211            self._add_to_public_id_map('kiwi_RWPart')
212        if 'kiwi_BootPart' not in self.public_partition_id_map:
213            self._add_to_public_id_map('kiwi_BootPart')
214        root_uuid = self.gUID.get('root')
215        if root_uuid:
216            self.partitioner.set_uuid(
217                self.partition_id_map['root'], root_uuid
218            )
219
220    def create_root_lvm_partition(
221        self, mbsize: str, clone: int = 0, partition_id: Optional[int] = None
222    ):
223        """
224        Create root partition for use with LVM
225
226        Populates kiwi_RootPart(id)
227
228        :param str mbsize: partition size string
229        :param int clone: create [clone] cop(y/ies) of the lvm roo partition
230        :param int partition_id:
231            If provided, use this exact partition ID
232            instead of auto-incrementing. When cloned, the clone
233            ID is calculated from the given partition_id
234        """
235        (mbsize, mbsize_clone) = Disk._parse_size(mbsize)
236        if clone:
237            self._create_clones(
238                'root', clone, 't.lvm', mbsize_clone, partition_id
239            )
240        self.partitioner.create(
241            name='p.lxlvm',
242            mbsize=mbsize,
243            type_name='t.lvm',
244            partition_id=partition_id
245        )
246        self._add_to_map('root')
247        self._add_to_public_id_map('kiwi_RootPart')
248        root_uuid = self.gUID.get('root')
249        if root_uuid:
250            self.partitioner.set_uuid(
251                self.partition_id_map['root'], root_uuid
252            )
253
254    def create_root_raid_partition(
255        self, mbsize: str, clone: int = 0, partition_id: Optional[int] = None
256    ):
257        """
258        Create root partition for use with MD Raid
259
260        Populates kiwi_RootPart(id) and kiwi_RaidPart(id) as well
261        as the default raid device node at boot time which is
262        configured to be kiwi_RaidDev(/dev/mdX)
263
264        :param str mbsize: partition size string
265        :param int clone: create [clone] cop(y/ies) of the raid root partition
266        :param int partition_id:
267            If provided, use this exact partition ID
268            instead of auto-incrementing. When cloned, the clone
269            ID is calculated from the given partition_id
270        """
271        (mbsize, mbsize_clone) = Disk._parse_size(mbsize)
272        if clone:
273            self._create_clones(
274                'root', clone, 't.raid', mbsize_clone, partition_id
275            )
276        self.partitioner.create(
277            name='p.lxraid',
278            mbsize=mbsize,
279            type_name='t.raid',
280            partition_id=partition_id
281        )
282        self._add_to_map('root')
283        self._add_to_public_id_map('kiwi_RootPart')
284        self._add_to_public_id_map('kiwi_RaidPart')
285        root_uuid = self.gUID.get('root')
286        if root_uuid:
287            self.partitioner.set_uuid(
288                self.partition_id_map['root'], root_uuid
289            )
290
291    def create_root_readonly_partition(
292        self, mbsize: str, clone: int = 0, partition_id: Optional[int] = None
293    ):
294        """
295        Create root readonly partition for use with overlayfs
296
297        Populates kiwi_ReadOnlyPart(id), the partition is meant to
298        contain a squashfs readonly filesystem. The partition size
299        should be the size of the squashfs filesystem in order to
300        avoid wasting disk space
301
302        :param str mbsize: partition size string
303        :param int clone: create [clone] cop(y/ies) of the ro root partition
304        :param int partition_id:
305            If provided, use this exact partition ID
306            instead of auto-incrementing. When cloned, the clone
307            ID is calculated from the given partition_id
308        """
309        (mbsize, mbsize_clone) = Disk._parse_size(mbsize)
310        if clone:
311            self._create_clones(
312                'root', clone, 't.linux', mbsize_clone, partition_id
313            )
314        self.partitioner.create(
315            name='p.lxreadonly',
316            mbsize=mbsize,
317            type_name='t.linux',
318            partition_id=partition_id
319        )
320        self._add_to_map('readonly')
321        self._add_to_public_id_map('kiwi_ROPart')
322        root_uuid = self.gUID.get('root')
323        if root_uuid:
324            self.partitioner.set_uuid(
325                self.partition_id_map['readonly'], root_uuid
326            )
327
328    def create_boot_partition(
329        self, mbsize: str, clone: int = 0, partition_id: Optional[int] = None
330    ):
331        """
332        Create boot partition
333
334        Populates kiwi_BootPart(id) and optional kiwi_BootPartClone(id)
335
336        :param str mbsize: partition size string
337        :param int clone: create [clone] cop(y/ies) of the boot partition
338        :param int partition_id:
339            If provided, use this exact partition ID
340            instead of auto-incrementing. When cloned, the clone
341            ID is calculated from the given partition_id
342        """
343        (mbsize, mbsize_clone) = Disk._parse_size(mbsize)
344        if clone:
345            self._create_clones(
346                'boot', clone, 't.linux', mbsize_clone, partition_id
347            )
348        self.partitioner.create(
349            name='p.lxboot',
350            mbsize=mbsize,
351            type_name='t.linux',
352            partition_id=partition_id
353        )
354        self._add_to_map('boot')
355        self._add_to_public_id_map('kiwi_BootPart')
356        boot_uuid = self.gUID.get('xbootldr')
357        if boot_uuid:
358            self.partitioner.set_uuid(
359                self.partition_id_map['boot'], boot_uuid
360            )
361
362    def create_prep_partition(
363        self, mbsize: str, partition_id: Optional[int] = None
364    ):
365        """
366        Create prep partition
367
368        Populates kiwi_PrepPart(id)
369
370        :param str mbsize: partition size string
371        :param int partition_id:
372            If provided, use this exact partition ID
373            instead of auto-incrementing.
374        """
375        (mbsize, _) = Disk._parse_size(mbsize)
376        self.partitioner.create(
377            name='p.prep',
378            mbsize=mbsize,
379            type_name='t.prep',
380            partition_id=partition_id
381        )
382        self._add_to_map('prep')
383        self._add_to_public_id_map('kiwi_PrepPart')
384
385    def create_spare_partition(
386        self, mbsize: str, partition_id: Optional[int] = None
387    ):
388        """
389        Create spare partition for custom use
390
391        Populates kiwi_SparePart(id)
392
393        :param str mbsize: partition size string
394        :param int partition_id:
395            If provided, use this exact partition ID
396            instead of auto-incrementing.
397        """
398        (mbsize, _) = Disk._parse_size(mbsize)
399        self.partitioner.create(
400            name='p.spare',
401            mbsize=mbsize,
402            type_name='t.linux',
403            partition_id=partition_id
404        )
405        self._add_to_map('spare')
406        self._add_to_public_id_map('kiwi_SparePart')
407
408    def create_swap_partition(
409        self, mbsize: str, partition_id: Optional[int] = None
410    ):
411        """
412        Create swap partition
413
414        Populates kiwi_SwapPart(id)
415
416        :param str mbsize: partition size string
417        :param int partition_id:
418            If provided, use this exact partition ID
419            instead of auto-incrementing.
420        """
421        (mbsize, _) = Disk._parse_size(mbsize)
422        self.partitioner.create(
423            name='p.swap',
424            mbsize=mbsize,
425            type_name='t.swap',
426            partition_id=partition_id
427        )
428        self._add_to_map('swap')
429        self._add_to_public_id_map('kiwi_SwapPart')
430        swap_uuid = self.gUID.get('swap')
431        if swap_uuid:
432            self.partitioner.set_uuid(
433                self.partition_id_map['swap'], swap_uuid
434            )
435
436    def create_efi_csm_partition(
437        self, mbsize: str, partition_id: Optional[int] = None
438    ):
439        """
440        Create EFI bios grub partition
441
442        Populates kiwi_BiosGrub(id)
443
444        :param str mbsize: partition size string
445        :param int partition_id:
446            If provided, use this exact partition ID
447            instead of auto-incrementing.
448        """
449        (mbsize, _) = Disk._parse_size(mbsize)
450        self.partitioner.create(
451            name='p.legacy',
452            mbsize=mbsize,
453            type_name='t.csm',
454            partition_id=partition_id
455        )
456        self._add_to_map('efi_csm')
457        self._add_to_public_id_map('kiwi_BiosGrub')
458
459    def create_efi_partition(
460        self, mbsize: str, partition_id: Optional[int] = None
461    ):
462        """
463        Create EFI partition
464
465        Populates kiwi_EfiPart(id)
466
467        :param str mbsize: partition size string
468        :param int partition_id:
469            If provided, use this exact partition ID
470            instead of auto-incrementing.
471        """
472        (mbsize, _) = Disk._parse_size(mbsize)
473        self.partitioner.create(
474            name='p.UEFI',
475            mbsize=mbsize,
476            type_name='t.efi',
477            partition_id=partition_id
478        )
479        self._add_to_map('efi')
480        self._add_to_public_id_map('kiwi_EfiPart')
481        esp_uuid = self.gUID.get('esp')
482        if esp_uuid:
483            self.partitioner.set_uuid(
484                self.partition_id_map['efi'], esp_uuid
485            )
486
487    def activate_boot_partition(self):
488        """
489        Activate boot partition
490
491        Note: not all Partitioner instances supports this
492        """
493        partition_id = None
494        if 'prep' in self.partition_id_map:
495            partition_id = self.partition_id_map['prep']
496        elif 'boot' in self.partition_id_map:
497            partition_id = self.partition_id_map['boot']
498        elif 'root' in self.partition_id_map:
499            partition_id = self.partition_id_map['root']
500
501        if partition_id:
502            self.partitioner.set_flag(partition_id, 'f.active')
503
504    def create_hybrid_mbr(self):
505        """
506        Turn partition table into a hybrid GPT/MBR table
507
508        Note: only GPT tables supports this
509        """
510        self.partitioner.set_hybrid_mbr()
511
512    def create_mbr(self):
513        """
514        Turn partition table into MBR (msdos table)
515
516        Note: only GPT tables supports this
517        """
518        self.partitioner.set_mbr()
519
520    def set_start_sector(self, start_sector: int):
521        """
522        Set start sector
523
524        Note: only effective on DOS tables
525        """
526        self.partitioner.set_start_sector(start_sector)
527
528    def wipe(self):
529        """
530        Zap (destroy) any GPT and MBR data structures if present
531        For DASD disks create a new VTOC table
532        """
533        if 'dasd' in self.table_type:
534            log.debug('Initialize DASD disk with new VTOC table')
535            fdasd_input = Temporary().new_file()
536            with open(fdasd_input.name, 'w') as vtoc:
537                vtoc.write('y\n\nw\nq\n')
538            bash_command = ' '.join(
539                [
540                    'cat', fdasd_input.name, '|',
541                    'fdasd', '-f', self.storage_provider.get_device()
542                ]
543            )
544            try:
545                Command.run(
546                    ['bash', '-c', bash_command]
547                )
548            except Exception:
549                # unfortunately fdasd reports that it can't read in the
550                # partition table which I consider a bug in fdasd. However
551                # the table was correctly created and therefore we continue.
552                # Problem is that we are not able to detect real errors
553                # with the fdasd operation at that point.
554                log.debug('potential fdasd errors were ignored')
555        else:
556            log.debug('Initialize %s disk', self.table_type)
557            Command.run(
558                [
559                    'sgdisk', '--zap-all', self.storage_provider.get_device()
560                ]
561            )
562
563    def map_partitions(self):
564        """
565        Map/Activate partitions
566
567        In order to access the partitions through a device node it is
568        required to map them if the storage provider is loop based
569        """
570        if self.storage_provider.is_loop():
571            if self.partition_mapper == 'kpartx':
572                Command.run(
573                    ['kpartx', '-s', '-a', self.storage_provider.get_device()]
574                )
575            else:
576                Command.run(
577                    ['partx', '--add', self.storage_provider.get_device()]
578                )
579            self.is_mapped = True
580        else:
581            Command.run(
582                ['partprobe', self.storage_provider.get_device()]
583            )
584
585    def get_public_partition_id_map(self) -> Dict[str, str]:
586        """
587        Populated partition name to number map
588        """
589        return OrderedDict(
590            sorted(self.public_partition_id_map.items())
591        )
592
593    def get_discoverable_partition_ids(self) -> Dict[str, str]:
594        """
595        Ask systemd for a list of standardized GUIDs for the
596        current architecture and return them in a dictionary.
597        If there is no such information available an empty
598        dictionary is returned
599
600        :return: key:value dict from systemd-id128
601
602        :rtype: dict
603        """
604        discoverable_ids = {}
605        try:
606            raw_lines = Command.run(
607                ['systemd-id128', 'show']
608            ).output.split(os.linesep)[1:]
609            for line in raw_lines:
610                if line:
611                    line = ' '.join(line.split())
612                    partition_name, uuid = line.split(' ')
613                    discoverable_ids[partition_name] = uuid
614        except KiwiError as issue:
615            log.warning(
616                f'Failed to obtain discoverable partition IDs: {issue}'
617            )
618            log.warning(
619                'Using built-in table'
620            )
621            discoverable_ids = Defaults.get_discoverable_partition_ids()
622        return discoverable_ids
623
624    def _create_clones(
625        self, name: str, clone: int, type_flag: str, mbsize: str,
626        partition_id: Optional[int] = None
627    ) -> None:
628        """
629        Create [clone] cop(y/ies) of the given partition name
630
631        The name of a clone partition uses the following name policy:
632
633        * {name}clone{id} for the partition name
634        * kiwi_{name}PartClone{id} for the kiwi map name
635
636        :param str name: basename to use for clone partition names
637        :param int clone: number of clones, >= 1
638        :param str type_flag: partition type name
639        :param str mbsize: partition size string
640        :param int partition_id:
641            If provided, use this exact partition ID to
642            calculate the clone ID with.
643        """
644        for clone_id in range(1, clone + 1):
645            if partition_id:
646                partition_id += 1
647            self.partitioner.create(
648                name=f'p.lx{name}clone{partition_id or clone_id}',
649                mbsize=mbsize,
650                type_name=type_flag,
651                partition_id=partition_id
652            )
653            self._add_to_map(f'{name}clone{partition_id or clone_id}')
654            self._add_to_public_id_map(
655                f'kiwi_{name}PartClone{partition_id or clone_id}'
656            )
657
658    @staticmethod
659    def _parse_size(value: str) -> Tuple[str, str]:
660        """
661        parse size value. This can be one of the following
662
663        * A number_string
664        * The string named: 'all_free'
665        * The string formatted as:
666              clone:{number_string_origin}:{number_string_clone}
667
668        The method returns a tuple for size and optional clone size
669        If no clone size exists both tuple values are the same
670
671        The given number_string for the size of the partition is
672        passed along to the actually used partitioner object and
673        expected to be valid there. In case invalid size information
674        is passed to the partitioner an exception will be raised
675        in the scope of the partitioner interface and the selected
676        partitioner class
677
678        :param str value: size value
679
680        :return: Tuple of strings
681
682        :rtype: tuple
683        """
684        if not format(value).startswith('clone:'):
685            return (value, value)
686        else:
687            size_list = value.split(':')
688            return (size_list[1], size_list[2])
689
690    def _add_to_public_id_map(self, name, value=None):
691        if not value:
692            value = self.partitioner.get_id()
693        self.public_partition_id_map[name] = value
694
695    def _add_to_map(self, name):
696        device_node = None
697        partition_number = format(self.partitioner.get_id())
698        if self.storage_provider.is_loop():
699            device_base = os.path.basename(self.storage_provider.get_device())
700            if self.partition_mapper == 'kpartx':
701                device_node = ''.join(
702                    ['/dev/mapper/', device_base, 'p', partition_number]
703                )
704            else:
705                device_node = ''.join(
706                    ['/dev/', device_base, 'p', partition_number]
707                )
708        else:
709            device = self.storage_provider.get_device()
710            if device[-1].isdigit():
711                device_node = ''.join(
712                    [device, 'p', partition_number]
713                )
714            else:
715                device_node = ''.join(
716                    [device, partition_number]
717                )
718        if device_node:
719            self.partition_map[name] = device_node
720            self.partition_id_map[name] = partition_number
721
722    def __exit__(self, exc_type, exc_value, traceback):
723        if self.storage_provider.is_loop() and self.is_mapped:
724            log.info('Cleaning up %s instance', type(self).__name__)
725            try:
726                if self.partition_mapper == 'kpartx':
727                    for device_node in self.partition_map.values():
728                        Command.run(['dmsetup', 'remove', device_node])
729                    Command.run(
730                        ['kpartx', '-d', self.storage_provider.get_device()]
731                    )
732                else:
733                    Command.run(
734                        ['partx', '--delete', self.storage_provider.get_device()]
735                    )
736            except Exception as issue:
737                log.error(
738                    'cleanup of partition maps on {} failed with: {}'.format(
739                        self.storage_provider.get_device(), issue
740                    )
741                )