vesion 2.1b1

This should help prevent installing a bunch of titles, only to find title.db doesn't exist.

LICENSE.md

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019 Ian Burgwin
+Copyright (c) 2019-2021 Ian Burgwin
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,7 +1,7 @@
 [![License](https://img.shields.io/badge/License-MIT-blue.svg)]() ![Releases](https://img.shields.io/github/downloads/ihaveamac/custom-install/total.svg)
 
 # custom-install
-Experimental script to automate the process of a manual title install for Nintendo 3DS. Originally created late June 2019.
+Installs a title directly to an SD card for the Nintendo 3DS. Originally created late June 2019.
 
 ## Summary
 

ci-gui.py

@@ -5,7 +5,7 @@
 # You can find the full license text in LICENSE.md in the root of this project.
 
 from os import environ, scandir
-from os.path import abspath, join, isfile, dirname
+from os.path import abspath, basename, dirname, join, isfile
 from sys import exc_info, platform
 from threading import Thread, Lock
 from time import strftime
@@ -18,11 +18,14 @@ from typing import TYPE_CHECKING
 
 from pyctr.crypto.engine import b9_paths
 from pyctr.util import config_dirs
+from pyctr.type.cdn import CDNError
+from pyctr.type.cia import CIAError
+from pyctr.type.tmd import TitleMetadataError
 
-from custominstall import CustomInstall
+from custominstall import CustomInstall, CI_VERSION, load_cifinish, InvalidCIFinishError
 
 if TYPE_CHECKING:
-    from typing import List
+    from typing import Dict, List
 
 is_windows = platform == 'win32'
 taskbar = None
@@ -37,7 +40,7 @@ if is_windows:
     except ModuleNotFoundError:
         pass
 
-file_parent = dirname(__file__)
+file_parent = dirname(abspath(__file__))
 
 # automatically load boot9 if it's in the current directory
 b9_paths.insert(0, join(file_parent, 'boot9.bin'))
@@ -52,6 +55,10 @@ except KeyError:
 seeddb_paths.insert(0, join(file_parent, 'seeddb.bin'))
 
 
+def clamp(n, smallest, largest):
+    return max(smallest, min(n, largest))
+
+
 def find_first_file(paths):
     for p in paths:
         if isfile(p):
@@ -94,6 +101,138 @@ class ConsoleFrame(ttk.Frame):
         self.text.configure(state=tk.DISABLED)
 
 
+def simple_listbox_frame(parent, title: 'str', items: 'List[str]'):
+    frame = ttk.LabelFrame(parent, text=title)
+    frame.rowconfigure(0, weight=1)
+    frame.columnconfigure(0, weight=1)
+
+    scrollbar = ttk.Scrollbar(frame, orient=tk.VERTICAL)
+    scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+
+    box = tk.Listbox(frame, highlightthickness=0, yscrollcommand=scrollbar.set, selectmode=tk.EXTENDED)
+    box.grid(row=0, column=0, sticky=tk.NSEW)
+    scrollbar.config(command=box.yview)
+
+    box.insert(tk.END, *items)
+
+    box.config(height=clamp(len(items), 3, 10))
+
+    return frame
+
+
+class TitleReadFailResults(tk.Toplevel):
+    def __init__(self, parent: tk.Tk = None, *, failed: 'Dict[str, str]'):
+        super().__init__(parent)
+        self.parent = parent
+
+        self.wm_withdraw()
+        self.wm_transient(self.parent)
+        self.grab_set()
+        self.wm_title('Failed to add titles')
+
+        self.rowconfigure(0, weight=1)
+        self.columnconfigure(0, weight=1)
+
+        outer_container = ttk.Frame(self)
+        outer_container.grid(sticky=tk.NSEW)
+        outer_container.rowconfigure(0, weight=0)
+        outer_container.rowconfigure(1, weight=1)
+        outer_container.columnconfigure(0, weight=1)
+
+        message_label = ttk.Label(outer_container, text="Some titles couldn't be added.")
+        message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
+
+        treeview_frame = ttk.Frame(outer_container)
+        treeview_frame.grid(row=1, column=0, sticky=tk.NSEW)
+        treeview_frame.rowconfigure(0, weight=1)
+        treeview_frame.columnconfigure(0, weight=1)
+
+        treeview_scrollbar = ttk.Scrollbar(treeview_frame, orient=tk.VERTICAL)
+        treeview_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+
+        treeview = ttk.Treeview(treeview_frame, yscrollcommand=treeview_scrollbar.set)
+        treeview.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        treeview.configure(columns=('filepath', 'reason'), show='headings')
+
+        treeview.column('filepath', width=200, anchor=tk.W)
+        treeview.heading('filepath', text='File path')
+        treeview.column('reason', width=400, anchor=tk.W)
+        treeview.heading('reason', text='Reason')
+
+        treeview_scrollbar.configure(command=treeview.yview)
+
+        for path, reason in failed.items():
+            treeview.insert('', tk.END, text=path, iid=path, values=(basename(path), reason))
+
+        ok_frame = ttk.Frame(outer_container)
+        ok_frame.grid(row=2, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        ok_frame.rowconfigure(0, weight=1)
+        ok_frame.columnconfigure(0, weight=1)
+
+        ok_button = ttk.Button(ok_frame, text='OK', command=self.destroy)
+        ok_button.grid(row=0, column=0)
+
+        self.wm_deiconify()
+
+
+class InstallResults(tk.Toplevel):
+    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool):
+        super().__init__(parent)
+        self.parent = parent
+
+        self.wm_withdraw()
+        self.wm_transient(self.parent)
+        self.grab_set()
+        self.wm_title('Install results')
+
+        self.rowconfigure(0, weight=1)
+        self.columnconfigure(0, weight=1)
+
+        outer_container = ttk.Frame(self)
+        outer_container.grid(sticky=tk.NSEW)
+        outer_container.rowconfigure(0, weight=0)
+        outer_container.columnconfigure(0, weight=1)
+
+        if install_state['failed'] and install_state['installed']:
+            # some failed and some worked
+            message = ('Some titles were installed, some failed. Please check the output for more details.\n'
+                       'The ones that were installed can be finished with custom-install-finalize.')
+        elif install_state['failed'] and not install_state['installed']:
+            # all failed
+            message = 'All titles failed to install. Please check the output for more details.'
+        elif install_state['installed'] and not install_state['failed']:
+            # all worked
+            message = 'All titles were installed.'
+        else:
+            message = 'Nothing was installed.'
+
+        if install_state['installed'] and copied_3dsx:
+            message += '\n\ncustom-install-finalize has been copied to the SD card.'
+
+        message_label = ttk.Label(outer_container, text=message)
+        message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
+
+        if install_state['installed']:
+            outer_container.rowconfigure(1, weight=1)
+            frame = simple_listbox_frame(outer_container, 'Installed', install_state['installed'])
+            frame.grid(row=1, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+
+        if install_state['failed']:
+            outer_container.rowconfigure(2, weight=1)
+            frame = simple_listbox_frame(outer_container, 'Failed', install_state['failed'])
+            frame.grid(row=2, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+
+        ok_frame = ttk.Frame(outer_container)
+        ok_frame.grid(row=3, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        ok_frame.rowconfigure(0, weight=1)
+        ok_frame.columnconfigure(0, weight=1)
+
+        ok_button = ttk.Button(ok_frame, text='OK', command=self.destroy)
+        ok_button.grid(row=0, column=0)
+
+        self.wm_deiconify()
+
+
 class CustomInstallGUI(ttk.Frame):
     console = None
 
@@ -101,6 +240,9 @@ class CustomInstallGUI(ttk.Frame):
         super().__init__(parent)
         self.parent = parent
 
+        # readers to give to CustomInstall at the install
+        self.readers = {}
+
         self.lock = Lock()
 
         self.log_messages = []
@@ -130,6 +272,16 @@ class CustomInstallGUI(ttk.Frame):
             f = fd.askdirectory(parent=parent, title='Select SD root (the directory or drive that contains '
                                                      '"Nintendo 3DS")', initialdir=file_parent, mustexist=True)
             if f:
+                cifinish_path = join(f, 'cifinish.bin')
+                try:
+                    load_cifinish(cifinish_path)
+                except InvalidCIFinishError:
+                    self.show_error(f'{cifinish_path} was corrupt!\n\n'
+                                    f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
+                                    f'It is also possible, though less likely, to be an issue with custom-install.\n\n'
+                                    f'Stopping now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
+                    return
+
                 sd_selected.delete('1.0', tk.END)
                 sd_selected.insert(tk.END, f)
 
@@ -179,53 +331,89 @@ class CustomInstallGUI(ttk.Frame):
 
         # ---------------------------------------------------------------- #
         # create buttons to add cias
-        listbox_buttons = ttk.Frame(self)
+        titlelist_buttons = ttk.Frame(self)
-        listbox_buttons.grid(row=1, column=0)
+        titlelist_buttons.grid(row=1, column=0)
 
         def add_cias_callback():
             files = fd.askopenfilenames(parent=parent, title='Select CIA files', filetypes=[('CIA files', '*.cia')],
                                         initialdir=file_parent)
+            results = {}
             for f in files:
-                self.add_cia(f)
+                success, reason = self.add_cia(f)
+                if not success:
+                    results[f] = reason
 
-        add_cias = ttk.Button(listbox_buttons, text='Add CIAs', command=add_cias_callback)
+            if results:
+                title_read_fail_window = TitleReadFailResults(self.parent, failed=results)
+                title_read_fail_window.focus()
+            self.sort_treeview()
+
+        add_cias = ttk.Button(titlelist_buttons, text='Add CIAs', command=add_cias_callback)
         add_cias.grid(row=0, column=0)
 
+        def add_cdn_callback():
+            d = fd.askdirectory(parent=parent, title='Select folder containing title contents in CDN format',
+                                initialdir=file_parent)
+            if d:
+                if isfile(join(d, 'tmd')):
+                    success, reason = self.add_cia(d)
+                    if not success:
+                        self.show_error(f"Couldn't add {basename(d)}: {reason}")
+                    else:
+                        self.sort_treeview()
+                else:
+                    self.show_error('tmd file not found in the CDN directory:\n' + d)
+
+        add_cdn = ttk.Button(titlelist_buttons, text='Add CDN title folder', command=add_cdn_callback)
+        add_cdn.grid(row=0, column=1)
+
         def add_dirs_callback():
             d = fd.askdirectory(parent=parent, title='Select folder containing CIA files', initialdir=file_parent)
             if d:
+                results = {}
                 for f in scandir(d):
                     if f.name.lower().endswith('.cia'):
-                        self.add_cia(f.path)
+                        success, reason = self.add_cia(f.path)
+                        if not success:
+                            results[f] = reason
 
-        add_dirs = ttk.Button(listbox_buttons, text='Add folder', command=add_dirs_callback)
+                if results:
-        add_dirs.grid(row=0, column=1)
+                    title_read_fail_window = TitleReadFailResults(self.parent, failed=results)
+                    title_read_fail_window.focus()
+                self.sort_treeview()
+
+        add_dirs = ttk.Button(titlelist_buttons, text='Add folder', command=add_dirs_callback)
+        add_dirs.grid(row=0, column=2)
 
         def remove_selected_callback():
-            indexes = self.cia_listbox.curselection()
+            for entry in self.treeview.selection():
-            n = 0
+                self.remove_cia(entry)
-            for i in indexes:
-                self.cia_listbox.delete(i - n)
-                n += 1
 
-        remove_selected = ttk.Button(listbox_buttons, text='Remove selected', command=remove_selected_callback)
+        remove_selected = ttk.Button(titlelist_buttons, text='Remove selected', command=remove_selected_callback)
-        remove_selected.grid(row=0, column=2)
+        remove_selected.grid(row=0, column=3)
 
         # ---------------------------------------------------------------- #
-        # create listbox
+        # create treeview
-        listbox_frame = ttk.Frame(self)
+        treeview_frame = ttk.Frame(self)
-        listbox_frame.grid(row=2, column=0, sticky=tk.NSEW)
+        treeview_frame.grid(row=2, column=0, sticky=tk.NSEW)
-        listbox_frame.rowconfigure(0, weight=1)
+        treeview_frame.rowconfigure(0, weight=1)
-        listbox_frame.columnconfigure(0, weight=1)
+        treeview_frame.columnconfigure(0, weight=1)
 
-        cia_listbox_scrollbar = ttk.Scrollbar(listbox_frame, orient=tk.VERTICAL)
+        treeview_scrollbar = ttk.Scrollbar(treeview_frame, orient=tk.VERTICAL)
-        cia_listbox_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+        treeview_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
 
-        self.cia_listbox = tk.Listbox(listbox_frame, highlightthickness=0, yscrollcommand=cia_listbox_scrollbar.set,
+        self.treeview = ttk.Treeview(treeview_frame, yscrollcommand=treeview_scrollbar.set)
-                                      selectmode=tk.EXTENDED)
+        self.treeview.grid(row=0, column=0, sticky=tk.NSEW)
-        self.cia_listbox.grid(row=0, column=0, sticky=tk.NSEW)
+        self.treeview.configure(columns=('filepath', 'titleid', 'titlename'), show='headings')
 
-        cia_listbox_scrollbar.config(command=self.cia_listbox.yview)
+        self.treeview.column('filepath', width=200, anchor=tk.W)
+        self.treeview.heading('filepath', text='File path')
+        self.treeview.column('titleid', width=50, anchor=tk.W)
+        self.treeview.heading('titleid', text='Title ID')
+        self.treeview.column('titlename', width=150, anchor=tk.W)
+        self.treeview.heading('titlename', text='Title name')
+
+        treeview_scrollbar.configure(command=self.treeview.yview)
 
         # ---------------------------------------------------------------- #
         # create progressbar
@@ -258,8 +446,7 @@ class CustomInstallGUI(ttk.Frame):
         self.status_label = ttk.Label(self, text='Waiting...')
         self.status_label.grid(row=5, column=0, sticky=tk.NSEW)
 
-        self.log('custom-install by ihaveamac', status=False)
+        self.log(f'custom-install {CI_VERSION} - https://github.com/ihaveamac/custom-install', status=False)
-        self.log('https://github.com/ihaveamac/custom-install', status=False)
 
         if is_windows and not taskbar:
             self.log('Note: comtypes module not found.')
@@ -269,9 +456,38 @@ class CustomInstallGUI(ttk.Frame):
 
         self.disable_during_install = (add_cias, add_dirs, remove_selected, start, *self.file_picker_textboxes.values())
 
+    def sort_treeview(self):
+        l = [(self.treeview.set(k, 'titlename'), k) for k in self.treeview.get_children()]
+        # sort by title name
+        l.sort(key=lambda x: x[0].lower())
+
+        for idx, pair in enumerate(l):
+            self.treeview.move(pair[1], '', idx)
+
     def add_cia(self, path):
         path = abspath(path)
-        self.cia_listbox.insert(tk.END, path)
+        if path in self.readers:
+            return False, 'File already in list'
+        try:
+            reader = CustomInstall.get_reader(path)
+        except (CIAError, CDNError, TitleMetadataError):
+            return False, 'Failed to read as a CIA or CDN title, probably corrupt'
+        except Exception as e:
+            return False, f'Exception occurred: {type(e).__name__}: {e}'
+
+        if reader.tmd.title_id.startswith('00048'):
+            return False, 'DSiWare is not supported'
+        try:
+            title_name = reader.contents[0].exefs.icon.get_app_title().short_desc
+        except:
+            title_name = '(No title)'
+        self.treeview.insert('', tk.END, text=path, iid=path, values=(path, reader.tmd.title_id, title_name))
+        self.readers[path] = reader
+        return True, ''
+
+    def remove_cia(self, path):
+        self.treeview.delete(path)
+        del self.readers[path]
 
     def open_console(self):
         if self.console:
@@ -345,23 +561,33 @@ class CustomInstallGUI(ttk.Frame):
                 return
 
         self.disable_buttons()
-        self.log('Starting install...')
 
-        cias = self.cia_listbox.get(0, tk.END)
+        if not len(self.readers):
-        if not len(cias):
             self.show_error('There are no titles added to install.')
             return
 
+        self.log('Starting install...')
+
+        if taskbar:
+            taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
+
         installer = CustomInstall(boot9=boot9,
                                   seeddb=seeddb,
                                   movable=movable_sed,
-                                  cias=cias,
                                   sd=sd_root,
                                   skip_contents=self.skip_contents_var.get() == 1,
                                   overwrite_saves=self.overwrite_saves_var.get() == 1)
 
+        # use the treeview which has been sorted alphabetically
+        #installer.readers = self.readers.values()
+        readers_final = []
+        for k in self.treeview.get_children():
+            readers_final.append(self.readers[self.treeview.set(k, 'filepath')])
+
+        installer.readers = readers_final
+
         finished_percent = 0
-        max_percentage = 100 * len(cias)
+        max_percentage = 100 * len(self.readers)
         self.progressbar.config(maximum=max_percentage)
 
         def ci_on_log_msg(message, *args, **kwargs):
@@ -393,23 +619,24 @@ class CustomInstallGUI(ttk.Frame):
         installer.event.on_error += ci_on_error
         installer.event.on_cia_start += ci_on_cia_start
 
-        if taskbar:
+        if self.skip_contents_var.get() != 1:
-            taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
+            total_size, free_space = installer.check_size()
+            if total_size > free_space:
+                self.show_error(f'Not enough free space.\n'
+                                f'Combined title install size: {total_size / (1024 * 1024):0.2f} MiB\n'
+                                f'Free space: {free_space / (1024 * 1024):0.2f} MiB')
+                self.enable_buttons()
+                return
 
         def install():
             try:
-                result, copied_3dsx = installer.start(continue_on_fail=False)
+                result, copied_3dsx = installer.start()
-                if result is True:
+                if result:
-                    self.log('Done!')
+                    result_window = InstallResults(self.parent, install_state=result, copied_3dsx=copied_3dsx)
-                    if copied_3dsx:
+                    result_window.focus()
-                        self.show_info('custom-install-finalize has been copied to the SD card.\n'
+                elif result is None:
-                                       'To finish the install, run this on the console through the homebrew launcher.\n'
+                    self.show_error("An error occurred when trying to run save3ds_fuse.\n"
-                                       'This will install a ticket and seed if required.')
+                                    "Either title.db doesn't exist, or save3ds_fuse couldn't be run.")
-                    else:
-                        self.show_info('To finish the install, run custom-install-finalize on the console.\n'
-                                       'This will install a ticket and seed if required.')
-                elif result is False:
-                    self.show_error('An error occurred when trying to run save3ds_fuse.')
                     self.open_console()
             except:
                 installer.event.on_error(exc_info())
@@ -420,7 +647,7 @@ class CustomInstallGUI(ttk.Frame):
 
 
 window = tk.Tk()
-window.title('custom-install')
+window.title(f'custom-install {CI_VERSION}')
 frame = CustomInstallGUI(window)
 frame.pack(fill=tk.BOTH, expand=True)
 window.mainloop()

custominstall.py

@@ -5,13 +5,15 @@
 # You can find the full license text in LICENSE.md in the root of this project.
 
 from argparse import ArgumentParser
-from os import makedirs, scandir
+from os import makedirs, rename, scandir
-from os.path import dirname, join, isfile
+from os.path import dirname, join, isdir, isfile
 from random import randint
 from hashlib import sha256
 from locale import getpreferredencoding
-from shutil import copyfile
+from pprint import pformat
+from shutil import copyfile, copy2, rmtree
 import sys
+from sys import platform, executable
 from tempfile import TemporaryDirectory
 from traceback import format_exception
 from typing import BinaryIO, TYPE_CHECKING
@@ -19,22 +21,34 @@ import subprocess
 
 if TYPE_CHECKING:
     from os import PathLike
-    from typing import Union
+    from typing import List, Union
 
 from events import Events
 
 from pyctr.crypto import CryptoEngine, Keyslot, load_seeddb, get_seed
-from pyctr.type.cia import CIAReader, CIASection
+from pyctr.type.cdn import CDNReader, CDNError
+from pyctr.type.cia import CIAReader, CIAError
 from pyctr.type.ncch import NCCHSection
+from pyctr.type.tmd import TitleMetadataError
 from pyctr.util import roundup
 
-is_windows = sys.platform == 'win32'
+if platform == 'msys':
+    platform = 'win32'
+
+is_windows = platform == 'win32'
+
+if is_windows:
+    from ctypes import c_wchar_p, pointer, c_ulonglong, windll
+else:
+    from os import statvfs
+
+CI_VERSION = '2.1b1'
 
 # used to run the save3ds_fuse binary next to the script
 frozen = getattr(sys, 'frozen', False)
 script_dir: str
 if frozen:
-    script_dir = dirname(sys.executable)
+    script_dir = dirname(executable)
 else:
     script_dir = dirname(__file__)
 
@@ -60,6 +74,26 @@ class InvalidCIFinishError(Exception):
     pass
 
 
+def get_free_space(path: 'Union[PathLike, bytes, str]'):
+    if is_windows:
+        lpSectorsPerCluster = c_ulonglong(0)
+        lpBytesPerSector = c_ulonglong(0)
+        lpNumberOfFreeClusters = c_ulonglong(0)
+        lpTotalNumberOfClusters = c_ulonglong(0)
+        ret = windll.kernel32.GetDiskFreeSpaceW(c_wchar_p(path), pointer(lpSectorsPerCluster),
+                                                pointer(lpBytesPerSector),
+                                                pointer(lpNumberOfFreeClusters),
+                                                pointer(lpTotalNumberOfClusters))
+        if not ret:
+            raise WindowsError
+        free_blocks = lpNumberOfFreeClusters.value * lpSectorsPerCluster.value
+        free_bytes = free_blocks * lpBytesPerSector.value
+    else:
+        stv = statvfs(path)
+        free_bytes = stv.f_bavail * stv.f_frsize
+    return free_bytes
+
+
 def load_cifinish(path: 'Union[PathLike, bytes, str]'):
     try:
         with open(path, 'rb') as f:
@@ -140,11 +174,24 @@ def save_cifinish(path: 'Union[PathLike, bytes, str]', data: dict):
             out.write(b''.join(finalize_entry_data))
 
 
+def get_install_size(title: 'Union[CIAReader, CDNReader]'):
+    sizes = [1] * 5
+
+    if title.tmd.save_size:
+        # one for the data directory, one for the 00000001.sav file
+        sizes.extend((1, title.tmd.save_size))
+
+    for record in title.content_info:
+        sizes.append(record.size)
+
+    # this calculates the size to put in the Title Info Entry
+    title_size = sum(roundup(x, TITLE_ALIGN_SIZE) for x in sizes)
+
+    return title_size
+
+
 class CustomInstall:
-
+    def __init__(self, boot9, seeddb, movable, sd, cifinish_out=None,
-    cia: CIAReader
-
-    def __init__(self, boot9, seeddb, movable, cias, sd, cifinish_out=None,
                  overwrite_saves=False, skip_contents=False):
         self.event = Events()
         self.log_lines = []  # Stores all info messages for user to view
@@ -152,7 +199,7 @@ class CustomInstall:
         self.crypto = CryptoEngine(boot9=boot9)
         self.crypto.setup_sd_key_from_file(movable)
         self.seeddb = seeddb
-        self.cias = cias
+        self.readers: 'List[Union[CDNReader, CIAReader]]' = []
         self.sd = sd
         self.skip_contents = skip_contents
         self.overwrite_saves = overwrite_saves
@@ -162,20 +209,62 @@ class CustomInstall:
     def copy_with_progress(self, src: BinaryIO, dst: BinaryIO, size: int, path: str, fire_event: bool = True):
         left = size
         cipher = self.crypto.create_ctr_cipher(Keyslot.SD, self.crypto.sd_path_to_iv(path))
+        hasher = sha256()
         while left > 0:
             to_read = min(READ_SIZE, left)
-            data = cipher.encrypt(src.read(READ_SIZE))
+            data = src.read(READ_SIZE)
-            dst.write(data)
+            hasher.update(data)
+            dst.write(cipher.encrypt(data))
             left -= to_read
             total_read = size - left
             if fire_event:
                 self.event.update_percentage((total_read / size) * 100, total_read / 1048576, size / 1048576)
 
-    def start(self, continue_on_fail=True):
+        return hasher.digest()
+
+    @staticmethod
+    def get_reader(path: 'Union[PathLike, bytes, str]'):
+        if isdir(path):
+            # try the default tmd file
+            reader = CDNReader(join(path, 'tmd'))
+        else:
+            try:
+                reader = CIAReader(path)
+            except CIAError:
+                # if there was an error with parsing the CIA header,
+                # the file would be tried in CDNReader next (assuming it's a tmd)
+                # any other error should be propagated to the caller
+                reader = CDNReader(path)
+        return reader
+
+    def prepare_titles(self, paths: 'List[PathLike]'):
+        readers = []
+        for path in paths:
+            self.log(f'Reading {path}')
+            try:
+                reader = self.get_reader(path)
+            except (CIAError, CDNError, TitleMetadataError):
+                self.log(f"Couldn't read {path}, likely corrupt or not a CIA or CDN title")
+                continue
+            if reader.tmd.title_id.startswith('00048'):  # DSiWare
+                self.log(f'Skipping {reader.tmd.title_id} - DSiWare is not supported')
+                continue
+            readers.append(reader)
+        self.readers = readers
+
+    def check_size(self):
+        total_size = 0
+        for r in self.readers:
+            total_size += get_install_size(r)
+
+        free_space = get_free_space(self.sd)
+        return total_size, free_space
+
+    def start(self):
         if frozen:
             save3ds_fuse_path = join(script_dir, 'bin', 'save3ds_fuse')
         else:
-            save3ds_fuse_path = join(script_dir, 'bin', sys.platform, 'save3ds_fuse')
+            save3ds_fuse_path = join(script_dir, 'bin', platform, 'save3ds_fuse')
         if is_windows:
             save3ds_fuse_path += '.exe'
         if not isfile(save3ds_fuse_path):
@@ -196,48 +285,71 @@ class CustomInstall:
             cifinish_path = self.cifinish_out
         else:
             cifinish_path = join(self.sd, 'cifinish.bin')
-        sd_path = join(sd_path, id1s[0])
+
-        title_info_entries = {}
+        try:
             cifinish_data = load_cifinish(cifinish_path)
+        except InvalidCIFinishError as e:
+            self.log(f'{type(e).__qualname__}: {e}')
+            self.log(f'{cifinish_path} was corrupt!\n'
+                     f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
+                     f'It is also possible, though less likely, to be an issue with custom-install.\n'
+                     f'Exiting now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
+            return None, False
+
+        with TemporaryDirectory(suffix='-custom-install') as tempdir:
+            # set up the common arguments for the two times we call save3ds_fuse
+            save3ds_fuse_common_args = [
+                save3ds_fuse_path,
+                '-b', crypto.b9_path,
+                '-m', self.movable,
+                '--sd', self.sd,
+                '--db', 'sdtitle',
+                tempdir
+            ]
+
+            extra_kwargs = {}
+            if is_windows:
+                # hide console window
+                extra_kwargs['creationflags'] = 0x08000000  # CREATE_NO_WINDOW
+
+            # extract the title database to add our own entry to
+            self.log('Extracting Title Database...')
+            out = subprocess.run(save3ds_fuse_common_args + ['-x'],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT,
+                                 encoding=getpreferredencoding(),
+                                 **extra_kwargs)
+            if out.returncode:
+                for l in out.stdout.split('\n'):
+                    self.log(l)
+                self.log('Command line:')
+                for l in pformat(out.args).split('\n'):
+                    self.log(l)
+                return None, False
+
+            sd_path = join(sd_path, id1s[0])
 
             load_seeddb(self.seeddb)
 
+            install_state = {'installed': [], 'failed': []}
+
             # Now loop through all provided cia files
-        
+            for idx, cia in enumerate(self.readers):
-        for idx, c in enumerate(self.cias):
-            self.log('Reading ' + c)
-
-            try:
-                cia = CIAReader(c)
-            except Exception as e:
-                self.event.on_error(sys.exc_info())
-                if continue_on_fail:
-                    continue
-                else:
-                    return None, False
 
                 self.event.on_cia_start(idx)
 
-            self.cia = cia
+                temp_title_root = join(self.sd, f'ci-install-temp-{cia.tmd.title_id}-{randint(0, 0xFFFFFFFF):08x}')
+                makedirs(temp_title_root, exist_ok=True)
 
                 tid_parts = (cia.tmd.title_id[0:8], cia.tmd.title_id[8:16])
 
                 try:
-                self.log(f'Installing {cia.contents[0].exefs.icon.get_app_title().short_desc}...')
+                    display_title = f'{cia.contents[0].exefs.icon.get_app_title().short_desc} - {cia.tmd.title_id}'
                 except:
-                self.log('Installing...')
+                    display_title = cia.tmd.title_id
+                self.log(f'Installing {display_title}...')
 
-            sizes = [1] * 5
+                title_size = get_install_size(cia)
-
-            if cia.tmd.save_size:
-                # one for the data directory, one for the 00000001.sav file
-                sizes.extend((1, cia.tmd.save_size))
-            
-            for record in cia.content_info:
-                sizes.append(record.size)
-            
-            # this calculates the size to put in the Title Info Entry
-            title_size = sum(roundup(x, TITLE_ALIGN_SIZE) for x in sizes)
 
                 # checks if this is dlc, which has some differences
                 is_dlc = tid_parts[0] == '0004008c'
@@ -258,6 +370,9 @@ class CustomInstall:
                 cmd_id = len(cia.content_info) if is_dlc else 1
                 cmd_filename = f'{cmd_id:08x}.cmd'
 
+                # this is where the final directory will be moved
+                tidhigh_root = join(sd_path, 'title', tid_parts[0])
+
                 # get the title root where all the contents will be
                 title_root = join(sd_path, 'title', *tid_parts)
                 content_root = join(title_root, 'content')
@@ -265,14 +380,16 @@ class CustomInstall:
                 title_root_cmd = f'/title/{"/".join(tid_parts)}'
                 content_root_cmd = title_root_cmd + '/content'
 
+                temp_content_root = join(temp_title_root, 'content')
+
                 if not self.skip_contents:
-                makedirs(join(content_root, 'cmd'), exist_ok=True)
+                    makedirs(join(temp_content_root, 'cmd'), exist_ok=True)
                     if cia.tmd.save_size:
-                    makedirs(join(title_root, 'data'), exist_ok=True)
+                        makedirs(join(temp_title_root, 'data'), exist_ok=True)
                     if is_dlc:
                         # create the separate directories for every 256 contents
                         for x in range(((len(cia.content_info) - 1) // 256) + 1):
-                        makedirs(join(content_root, f'{x:08x}'), exist_ok=True)
+                            makedirs(join(temp_content_root, f'{x:08x}'), exist_ok=True)
 
                     # maybe this will be changed in the future
                     tmd_id = 0
@@ -280,46 +397,58 @@ class CustomInstall:
                     tmd_filename = f'{tmd_id:08x}.tmd'
 
                     # write the tmd
-                enc_path = content_root_cmd + '/' + tmd_filename
+                    tmd_enc_path = content_root_cmd + '/' + tmd_filename
-                self.log(f'Writing {enc_path}...')
+                    self.log(f'Writing {tmd_enc_path}...')
-                with cia.open_raw_section(CIASection.TitleMetadata) as s:
+                    with open(join(temp_content_root, tmd_filename), 'wb') as o:
-                    with open(join(content_root, tmd_filename), 'wb') as o:
+                        with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv(tmd_enc_path)) as e:
-                        self.copy_with_progress(s, o, cia.sections[CIASection.TitleMetadata].size, enc_path,
+                            e.write(bytes(cia.tmd))
-                                                fire_event=False)
 
+                    # in case the contents are corrupted
+                    do_continue = False
                     # write each content
                     for co in cia.content_info:
                         content_filename = co.id + '.app'
                         if is_dlc:
                             dir_index = format((co.cindex // 256), '08x')
-                        enc_path = content_root_cmd + f'/{dir_index}/{content_filename}'
+                            content_enc_path = content_root_cmd + f'/{dir_index}/{content_filename}'
-                        out_path = join(content_root, dir_index, content_filename)
+                            content_out_path = join(temp_content_root, dir_index, content_filename)
                         else:
-                        enc_path = content_root_cmd + '/' + content_filename
+                            content_enc_path = content_root_cmd + '/' + content_filename
-                        out_path = join(content_root, content_filename)
+                            content_out_path = join(temp_content_root, content_filename)
-                    self.log(f'Writing {enc_path}...')
+                        self.log(f'Writing {content_enc_path}...')
-                    with cia.open_raw_section(co.cindex) as s, open(out_path, 'wb') as o:
+                        with cia.open_raw_section(co.cindex) as s, open(content_out_path, 'wb') as o:
-                        self.copy_with_progress(s, o, co.size, enc_path)
+                            result_hash = self.copy_with_progress(s, o, co.size, content_enc_path)
+                            if result_hash != co.hash:
+                                self.log(f'WARNING: Hash does not match for {content_enc_path}!')
+                                install_state['failed'].append(display_title)
+                                rename(temp_title_root, temp_title_root + '-corrupted')
+                                do_continue = True
+                                break
+
+                    if do_continue:
+                        continue
 
                     # generate a blank save
                     if cia.tmd.save_size:
-                    enc_path = title_root_cmd + '/data/00000001.sav'
+                        sav_enc_path = title_root_cmd + '/data/00000001.sav'
-                    out_path = join(title_root, 'data', '00000001.sav')
+                        tmp_sav_out_path = join(temp_title_root, 'data', '00000001.sav')
-                    if self.overwrite_saves or not isfile(out_path):
+                        sav_out_path = join(title_root, 'data', '00000001.sav')
-                        cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(enc_path))
+                        if self.overwrite_saves or not isfile(sav_out_path):
+                            cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(sav_enc_path))
                             # in a new save, the first 0x20 are all 00s. the rest can be random
                             data = cipher.encrypt(b'\0' * 0x20)
-                        self.log(f'Generating blank save at {enc_path}...')
+                            self.log(f'Generating blank save at {sav_enc_path}...')
-                        with open(out_path, 'wb') as o:
+                            with open(tmp_sav_out_path, 'wb') as o:
                                 o.write(data)
                                 o.write(b'\0' * (cia.tmd.save_size - 0x20))
                         else:
-                        self.log(f'Not overwriting existing save at {enc_path}')
+                            self.log(f'Copying original save file from {sav_enc_path}...')
+                            copy2(sav_out_path, tmp_sav_out_path)
 
                     # generate and write cmd
-                enc_path = content_root_cmd + '/cmd/' + cmd_filename
+                    cmd_enc_path = content_root_cmd + '/cmd/' + cmd_filename
-                out_path = join(content_root, 'cmd', cmd_filename)
+                    cmd_out_path = join(temp_content_root, 'cmd', cmd_filename)
-                self.log(f'Generating {enc_path}')
+                    self.log(f'Generating {cmd_enc_path}')
                     highest_index = 0
                     content_ids = {}
 
@@ -366,9 +495,9 @@ class CustomInstall:
                     final += b''.join(installed_ids)
                     final += b''.join(cmacs)
 
-                cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(enc_path))
+                    cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(cmd_enc_path))
-                self.log(f'Writing {enc_path}')
+                    self.log(f'Writing {cmd_enc_path}')
-                with open(out_path, 'wb') as o:
+                    with open(cmd_out_path, 'wb') as o:
                         o.write(cipher.encrypt(final))
 
                 # this starts building the title info entry
@@ -405,46 +534,20 @@ class CustomInstall:
                     b'\0' * 0x2c
                 ]
 
-            title_info_entries[cia.tmd.title_id] = b''.join(title_info_entry_data)
+                if isdir(title_root):
+                    self.log(f'Removing original install at {title_root}...')
+                    rmtree(title_root)
+
+                makedirs(tidhigh_root, exist_ok=True)
+                rename(temp_title_root, title_root)
 
                 cifinish_data[int(cia.tmd.title_id, 16)] = {'seed': (get_seed(cia.contents[0].program_id) if cia.contents[0].flags.uses_seed else None)}
 
                 # This is saved regardless if any titles were installed, so the file can be upgraded just in case.
                 save_cifinish(cifinish_path, cifinish_data)
 
-        if title_info_entries:
+                with open(join(tempdir, cia.tmd.title_id), 'wb') as o:
-            with TemporaryDirectory(suffix='-custom-install') as tempdir:
+                    o.write(b''.join(title_info_entry_data))
-                # set up the common arguments for the two times we call save3ds_fuse
-                save3ds_fuse_common_args = [
-                    save3ds_fuse_path,
-                    '-b', crypto.b9_path,
-                    '-m', self.movable,
-                    '--sd', self.sd,
-                    '--db', 'sdtitle',
-                    tempdir
-                ]
-
-                extra_kwargs = {}
-                if is_windows:
-                    # hide console window
-                    extra_kwargs['creationflags'] = 0x08000000  # CREATE_NO_WINDOW
-
-                # extract the title database to add our own entry to
-                self.log('Extracting Title Database...')
-                out = subprocess.run(save3ds_fuse_common_args + ['-x'],
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.STDOUT,
-                                     encoding=getpreferredencoding(),
-                                     **extra_kwargs)
-                if out.returncode:
-                    for l in out.stdout.split('\n'):
-                        self.log(l)
-                    return False
-
-                for title_id, entry in title_info_entries.items():
-                    # write the title info entry to the temp directory
-                    with open(join(tempdir, title_id), 'wb') as o:
-                        o.write(entry)
 
                 # import the directory, now including our title
                 self.log('Importing into Title Database...')
@@ -456,12 +559,18 @@ class CustomInstall:
                 if out.returncode:
                     for l in out.stdout.split('\n'):
                         self.log(l)
-                    return False, False
+                    self.log('Command line:')
+                    for l in pformat(out.args).split('\n'):
+                        self.log(l)
+                    install_state['failed'].append(display_title)
 
+                install_state['installed'].append(display_title)
+
+            copied = False
+            if install_state['installed']:
                 finalize_3dsx_orig_path = join(script_dir, 'custom-install-finalize.3dsx')
                 hb_dir = join(self.sd, '3ds')
                 finalize_3dsx_path = join(hb_dir, 'custom-install-finalize.3dsx')
-            copied = False
                 if isfile(finalize_3dsx_orig_path):
                     self.log('Copying finalize program to ' + finalize_3dsx_path)
                     makedirs(hb_dir, exist_ok=True)
@@ -473,11 +582,8 @@ class CustomInstall:
                 self.log('This will install a ticket and seed if required.')
                 if copied:
                     self.log('custom-install-finalize has been copied to the SD card.')
-            return True, copied
 
-        else:
+            return install_state, copied
-            self.log('Did not install any titles.', 2)
-            return None, False
 
     def get_sd_path(self):
         sd_path = join(self.sd, 'Nintendo 3DS', self.crypto.id0.hex())
@@ -522,7 +628,7 @@ class CustomInstall:
 
 
 if __name__ == "__main__":
-    parser = ArgumentParser(description='Manually install a CIA to the SD card for a Nintendo 3DS system.')
+    parser = ArgumentParser(description='Install a CIA to the SD card for a Nintendo 3DS system.')
     parser.add_argument('cia', help='CIA files', nargs='+')
     parser.add_argument('-m', '--movable', help='movable.sed file', required=True)
     parser.add_argument('-b', '--boot9', help='boot9 file')
@@ -532,11 +638,11 @@ if __name__ == "__main__":
     parser.add_argument('--overwrite-saves', help='overwrite existing save files', action='store_true')
     parser.add_argument('--cifinish-out', help='path for cifinish.bin file, defaults to (SD root)/cifinish.bin')
 
+    print(f'custom-install {CI_VERSION} - https://github.com/ihaveamac/custom-install')
     args = parser.parse_args()
 
     installer = CustomInstall(boot9=args.boot9,
                               seeddb=args.seeddb,
-                              cias=args.cia,
                               movable=args.movable,
                               sd=args.sd,
                               overwrite_saves=args.overwrite_saves,
@@ -558,7 +664,17 @@ if __name__ == "__main__":
     installer.event.update_percentage += percent_handle
     installer.event.on_error += error
 
-    result, copied_3dsx = installer.start(continue_on_fail=False)
+    installer.prepare_titles(args.cia)
+
+    if not args.skip_contents:
+        total_size, free_space = installer.check_size()
+        if total_size > free_space:
+            installer.event.on_log_msg(f'Not enough free space.\n'
+                                       f'Combined title install size: {total_size / (1024 * 1024):0.2f} MiB\n'
+                                       f'Free space: {free_space / (1024 * 1024):0.2f} MiB')
+            sys.exit(1)
+
+    result, copied_3dsx = installer.start()
     if result is False:
         # save3ds_fuse failed
         installer.log('NOTE: Once save3ds_fuse is fixed, run the same command again with --skip-contents')

finalize/source/main.c

@@ -189,6 +189,7 @@ int load_cifinish(char* path, struct finish_db_entry_final **entries)
 		}
 	}
 
+	fclose(fp);
 	return header.title_count;
 
 fail:
@@ -217,9 +218,6 @@ void finalize_install(void)
 		return;
 	}
 
-	//printf("Deleting %s...\n", CIFINISH_PATH);
-	//unlink(CIFINISH_PATH);
-
 	memcpy(&ticket_buf, basetik_bin, basetik_bin_size);
 
 	for (int i = 0; i < title_count; ++i)
@@ -266,6 +264,9 @@ void finalize_install(void)
 		}
 	}
 
+	printf("Deleting %s...\n", CIFINISH_PATH);
+	unlink(CIFINISH_PATH);
+
 	free(entries);
 }
 
@@ -275,7 +276,7 @@ int main(int argc, char* argv[])
 	gfxInitDefault();
 	consoleInit(GFX_TOP, NULL);
 
-	printf("custom-install-finalize v1.4\n");
+	printf("custom-install-finalize v1.5\n");
 
 	finalize_install();
 	// print this at the end in case it gets pushed off the screen

requirements.txt

@@ -1,3 +1,2 @@
-pycryptodomex==3.9.8
+events==0.4
-events==0.3
+pyctr==0.4.5
-pyctr==0.4.1